stashing all changes

README.md

@@ -3,7 +3,7 @@ StilLett API Service
 
 The original `SAPIS` (Fahlborg and Rennes, 2016), was a restful web service based on Java Spring, implementing an API with 
 the ability to interpret options and input data as variables in an input JSON 
-object, passed to the `SAPIS` service in a HTTP request . The serivce has since then been rewritten in Python3, with similar functionality.
+object, passed to the `SAPIS` service in a HTTP request. Since then, the service has been rewritten in Python twice. This current version parses the text using `spaCy` rather than the original `efselab` for speed. 
 
 The API service can be reached from a server located at: https://sapis.it.liu.se/api, accepting POST requests.
 

pipeline.py

@@ -321,9 +321,9 @@ def split_measures_to_scream_headings(structural_instance) -> dict:
         "verbial_root_ratio",  # VerbalRoots        OK?
         "avg_verbal_arity",  # AVA                  OK?
         "verb_arity_unigram_probs",  # UVA          OK?
-        "calculate_avg_word_per_clause",  # TPC
-        "dep_unigram_probs",  # UnigramDep
-        "pos_unigram_probs",  # UnigramPOS
+        "calculate_avg_word_per_clause",  # TPC     OK
+        "dep_unigram_probs",  # UnigramDep          OK
+        "pos_unigram_probs",  # UnigramPOS          OK
         "lexical_density",  # RatCont               OK?
     ]
     surface_vars = ["avg_sentence_length", "avg_word_length", "avg_n_syllables"]  # ok
@@ -334,8 +334,8 @@ def split_measures_to_scream_headings(structural_instance) -> dict:
         "swevoc_h_ratio",  # ok?
     ]
     additional_structural = [
-        "n_verbal_roots",
-        "n_prep_comp",
+        "n_verbal_roots",  # ok
+        "n_prep_comp",  # ok
     ]
     additional_lexical = [  # Raw freqs
         "n_swevoc_s",  # ok

scream2/README.md

@@ -27,8 +27,45 @@ Below follows a brief explanation of all metrics calculated by `SCREAM2`. These
 
 ### Lexical metrics
 
-#### Additional surface metrics
+- `swevoc_total_ratio`: placeholder
+- `swevoc_c_ratio`: placeholder
+- `swevoc_d_ratio`: placeholder
+- `swevoc_h_ratio`: placeholder
+
+#### Additional lexical metrics
+
+- `n_swevoc_s`: placeholder
+- `n_swevoc_k`: placeholder
+- `n_swevoc_total`: placeholder
+- `n_swevoc_c`: placeholder
+- `n_swevoc_d`: placeholder
+- `n_swevoc_h`: placeholder
+- `swevoc_s_ratio`: placeholder
+- `swevoc_k_ratio`: placeholder
+- `swevoc_dict`: The dictionary containing all occurrences of SweVoc-words from the provided document. ALl lexical metrics are then calculated from these values. Primarily useful for debugging and can safetly be commented out from a production build.
 
 ### Structural vars
 
-#### Additional structural vars
+- `avg_dep_distance_dependent`: placeholder
\ No newline at end of file
+- `avg_dep_distance_sentence`: placeholder
+- `avg_right_dep_ratio`: placeholder
+- `avg_nominal_premodifiers`: placeholder
+- `avg_nominal_postmodifiers`: placeholder
+- `avg_prep_comp`: placeholder
+- `avg_sentence_depth`: placeholder
+- `verbial_root_ratio`: placeholder
+- `avg_verb_arity`: placeholder
+- `verb_arities_unigram_probs`: placeholder (namechange? not unigram now I think)
+- `avg_words_per_clause`: placeholder
+- `dep_ud_probs`: placeholder
+- `pos_ud_probs`: placeholder
+- `lexical_density`: placeholder
+
+#### Additional structural vars
+
+- `n_verbs`: placeholder (?)
+- `total_verb_arity`: placeholder (?)
+- `total_dep_distance`: placeholder (?)
+- `n_verbial_roots`: placeholder
+- `verb_arities_dict`: placeholder (?)
+- `n_prep_comp`: placeholder

scream2/SweVoc/swevoc.py

@@ -37,6 +37,34 @@ PAROLE_TO_SUC_CONVERTION = {
     "NCN": "NN",
 }
 
+# Simple table for conversion from SUC tags to UPOS tags
+SUC_TO_UPOS = {
+    "JJ": "ADJ",
+    "PP": "ADP",
+    "AB": "ADV",
+    "VB": "VERB",
+    "KN": "CCONJ",
+    "DT": "DET",
+    "IE": "INTJ",
+    "NN": "NOUN",
+    "RG": "NUM",
+    "PC": "PART",
+    "PN": "PRON",
+    "PM": "PROPN",
+    "MAD": "SYM",
+    "SN": "SCONJ",
+    "XX": "X",
+    # Some tags are missing here
+    "HS": "PRON",  # Assuming HS (Possessive Pronoun) should be mapped to PRON
+    "HP": "PRON",  # Assuming HP (Interrogative/Relative Pronoun) should be mapped to PRON
+    "PL": "PRON",  # Assuming PL (Interrogative/Relative Pronoun) should be mapped to PRON
+    "HD": "ADV",  # Assuming HD (Interrogative/Relative Adverb) should be mapped to ADV
+    "IN": "INTJ",  # Assuming IN (Interjection) should be mapped to INTJ
+    "HA": "ADV",  # Assuming HA (Interrogative/Relative Adverb) should be mapped to ADV
+    "PS": "PRON",  # Assuming PS (Possessive Pronoun) should be mapped to PRON
+    "RO": "NUM",  # Assuming RO (Ordinal Numeral) should be mapped to NUM
+}
+
 
 def parse_swe_voc():
     """
@@ -58,7 +86,9 @@ def parse_swe_voc():
         with open(path, "r", encoding=ENCODING) as f:
             for line in f.readlines():
                 split_line = line.strip().split("\t")
-                tag = PAROLE_TO_SUC_CONVERTION[split_line[1]]
+                tag = SUC_TO_UPOS[
+                    PAROLE_TO_SUC_CONVERTION[split_line[1]]
+                ]  # dirty line, fix it later
                 word = split_line[2]
                 categories = [category.strip() for category in split_line[3].split(",")]
 

scream2/configuration.py

-approved_data_model_types = {"float", "int", "list", "dict"}
-
 # Metrics
-QUOTATION_MARKS = {"'", '"', "“", "”", "‘", "’"}
 VOWELS = set("aeiouyåäöAEIOUYÅÄÖ")
 LIX_LIMIT = 6
 SPECIAL_CHARS = [",", ":", ".", "!", '"', "'"]
-rules = ["svo", "p2a", "prox", "qi", "split_a", "split_r", "split_k"]
 
-# Simple table for conversion from UPOS tags to SUC tags
-UPOS_TO_SUC = {
-    "ADJ": "JJ",  # adjective
-    "ADP": "PP",  # adposition
-    "ADV": "AB",  # adverb
-    "AUX": "VB",  # auxiliary
-    "CCONJ": "KN",  # coordinating conjunction
-    "DET": "DT",  # determiner
-    "INTJ": "IE",  # interjection
-    "NOUN": "NN",  # noun
-    "NUM": "RG",  # numeral
-    "PART": "PC",  # particle
-    "PRON": "PN",  # pronoun
-    "PROPN": "PM",  # proper noun
-    "PUNCT": "MAD",  # punctuation
-    "SCONJ": "SN",  # subordinating conjunction
-    "SYM": "MAD",  # symbol
-    "VERB": "VB",  # verb
-    "X": "XX",  # other
+UPOS_POS_TYPES = {
+    "ADJ": 0,  # Adjective
+    "ADP": 0,  # Adposition
+    "ADV": 0,  # Adverb
+    "AUX": 0,  # Auxiliary verb
+    "CCONJ": 0,  # Coordinating conjunction
+    "DET": 0,  # Determiner
+    "INTJ": 0,  # Interjection
+    "NOUN": 0,  # Noun
+    "NUM": 0,  # Numeral
+    "PART": 0,  # Particle
+    "PRON": 0,  # Pronoun
+    "PROPN": 0,  # Proper noun
+    "PUNCT": 0,  # Punctuation
+    "SCONJ": 0,  # Subordinating conjunction
+    "SYM": 0,  # Symbol
+    "VERB": 0,  # Verb
+    "X": 0,  # Other
 }
 
-# Part-of-speech related settings
-POS_TAGS = [
-    "AB",
-    "CITE",
-    "DT",
-    "HA",
-    "HD",
-    "HP",
-    "HS",
-    "IE",
-    "IN",
-    "JJ",
-    "KN",
-    "MAD",
-    "MID",
-    "NN",
-    "PAD",
-    "PC",
-    "PL",
-    "PLQS",
-    "PM",
-    "PN",
-    "PP",
-    "PS",
-    "RG",
-    "RO",
-    "SN",
-    "UO",
-    "VB",
-]
-
-VERB_TAG = "VERB"
-CONTENT_WORD_TAGS = {"NOUN", "ADJ", "ADV"}
-nominal_components = {"NOUN", "ADP", "PART"}
-verb_components = {"PRON", "ADV", "VERB"}
-# punctuation_marks = {"MAD", "MID", "PAD"} Managed by token.is_punct
-# word_pos_tags = {tag for tag in POS_TAGS if tag not in punctuation_marks}
-
-# Dependency related settings
-dep_types = [
-    "++",
-    "+A",
-    "+F",
-    "AA",
-    "AG",
-    "AN",
-    "AT",
-    "CA",
-    "CJ",
-    "DB",
-    "DT",
-    "EF",
-    "EO",
-    "ES",
-    "ET",
-    "FO",
-    "FP",
-    "FS",
-    "FV",
-    "HD",
-    "I?",
-    "IC",
-    "IF",
-    "IG",
-    "IK",
-    "IO",
-    "IP",
-    "IQ",
-    "IR",
-    "IS",
-    "IT",
-    "IU",
-    "IV",
-    "JC",
-    "JG",
-    "JR",
-    "JT",
-    "KA",
-    "MA",
-    "MS",
-    "NA",
-    "OA",
-    "OO",
-    "OP",
-    "PA",
-    "PL",
-    "PT",
-    "RA",
-    "ROOT",
-    "SP",
-    "SS",
-    "TA",
-    "UA",
-    "VA",
-    "VG",
-    "VO",
-    "VS",
-    "XA",
-    "XF",
-    "XT",
-    "XX",
-    "YY",
-]
-
-DEP_REL_TYPES = [
-    "nsubj",  # Subject
-    "obj",  # Object
-    "iobj",  # Indirect Object
-    "obl",  # Oblique nominal
-    "amod",  # Nominal (adjectival) pre-modifier
-    "det",  # Determiner
-    "nmod",  # Nominal modifier
-    "cc",  # Coordinating conjunction
-    "conj",  # Conjunct (in coordinate structure)
-    "advmod",  # Adverbial modifier
-    "appos",  # Apposition
-    "case",  # Preposition
-    "ccomp",  # Clausal complement
-    "obj",  # Logical object (same as obj?)
-    "nsubj:pass",  # Passive subject
-    "parataxis",  # Parataxis
-    "punct",  # Punctuation
-    "root",  # Root
-    "mark",  # Marker
-    "advcl",  # Adverbial clause modifier
-    "cop",  # Copula
-    "csubj",  # Clausal subject
-    "expl",  # Expletive (Logical subject?)
-    "list",  # List
-    "compound",  # Compound
-    "dislocated",  # Dislocated
-    "vocative",  # Vocative
-    "aux",  # Auxiliary
-    "discourse",  # Discourse
-    "punct",  # Punctuation
-    "reparandum",  # Overridden disfluency
-    "flat",  # Flat multiword expression
-    "cop",  # Copula
-    "xcomp",  # Open clausal complement
-    "iobj",  # Indirect object
-    "fixed",  # Fixed multiword expression
-    "orphan",  # Orphan
-    "dep",  # Unspecified dependency
-    "advmod",  # Conjunctional adverbial
-    "cc",  # Coordination at main clause level
-    "advmod",  # Other adverbial
-    "agent",  # Agent
-    "advmod",  # Contrastive adverbial
-    "dep",  # Doubled function (unspecified)
-    "acl:relcl",  # Relative clause in cleft
-    "obj",  # Logical object
-    "obj",  # Dummy object
-    "xcomp",  # Free subjective predicative complement
-    "expl",  # Dummy subject
-    "root",  # Finite predicate verb
-    "punct",  # Question mark
-    "punct",  # Quotation mark
-    "intj",  # Interjection phrase
-    "root",  # Head
-    "xcomp",  # Infinitive verb phrase minus infinitive marker
-    "acl",  # Subordinate clause minus subordinating conjunction
-    "compound:prt",  # Verb group
-]
+DEP_UD_TYPES = {
+    "nsubj": 0,
+    "obj": 0,
+    "iobj": 0,
+    "obl": 0,
+    "amod": 0,
+    "det": 0,
+    "nmod": 0,
+    "cc": 0,
+    "conj": 0,
+    "advmod": 0,
+    "appos": 0,
+    "case": 0,
+    "ccomp": 0,
+    "nsubj:pass": 0,
+    "parataxis": 0,
+    "punct": 0,
+    "root": 0,
+    "mark": 0,
+    "advcl": 0,
+    "cop": 0,
+    "csubj": 0,
+    "expl": 0,
+    "list": 0,
+    "compound": 0,
+    "dislocated": 0,
+    "vocative": 0,
+    "aux": 0,
+    "discourse": 0,
+    "reparandum": 0,
+    "flat": 0,
+    "xcomp": 0,
+    "fixed": 0,
+    "orphan": 0,
+    "dep": 0,
+    "agent": 0,
+    "acl:relcl": 0,
+    "intj": 0,
+    "acl": 0,
+    "compound:prt": 0,
+}
 
 SUBCLAUSE_DEP = "acl"  # UA
 POSTMODIFIER_DEP = "conj"  # ET
@@ -198,5 +71,5 @@ PREMODIFIER_DEP = "amod"  # AT
 PRECOMP_DEP = "obl"  # PA
 ROOT = "ROOT"
 
-# Verb arity related settings
-verb_arities = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9}
+VERB_TAG = "VERB"
+CONTENT_WORD_TAGS = {"NOUN", "ADJ", "ADV"}

scream2/scream2.py

@@ -8,10 +8,8 @@ in this code. Variable declaration as well as calculations are all separated
 into their own sections for readability and maintainability.
 """
 
-import spacy
-import spacy.attrs
 from spacy.language import Language
-from spacy.tokens import Doc
+from spacy.tokens import Doc, Token
 
 from scream2.SweVoc.swevoc import load_swe_voc
 
@@ -49,7 +47,20 @@ def scream_metrics(doc: Doc) -> Doc:
     n_unique_tokens = 0
     n_unique_words = 0
 
-    n_punctuations = 0
+    def add_probs(token: Token) -> None:
+        """
+        Helper function to add probabilities to the UPOS_POS_TYPES and DEP_UD_TYPES dictionaries.
+        :param token: The token to add probabilities for.
+        """
+        try:
+            config.UPOS_POS_TYPES[token.pos_] += 1
+        except KeyError:
+            pass
+
+        try:
+            config.DEP_UD_TYPES[token.dep_] += 1
+        except KeyError:
+            pass
 
     # -- Lexical metrics --
 
@@ -84,7 +95,7 @@ def scream_metrics(doc: Doc) -> Doc:
     n_content_words = 0
     n_pos_tags = 0
     n_verbs = 0
-    n_verbial_root = 0
+    n_verbal_roots = 0
     n_syllables = 0
 
     n_sub_clauses = 0
@@ -95,10 +106,13 @@ def scream_metrics(doc: Doc) -> Doc:
     n_prep_comp = 0
     n_right_dependencies = 0
     total_dep_distance = 0
-    # n_dependencies = 0 NOTE: Seems to equal the amount of tokens in the document
+    # n_dependencies = 0 NOTE: Seems to equal the amount of tokens in the document -- minus the root
     # n_dep_tags = 0 NOTE: Seems to equal the amount of tokens in the document
     sentence_depth = 0
 
+    dep_ud_probs = {}
+    pos_ud_probs = {}
+
     def get_sentence_depth(token) -> float:
         """
         Helper function to calculate the depth of a sentence.
@@ -126,7 +140,14 @@ def scream_metrics(doc: Doc) -> Doc:
             n_unique_tokens += 1
             unique_tokens.add(token)
 
-        if token.is_alpha:
+        add_probs(token)
+
+        if token.dep_ != config.ROOT:
+            dep_distance = token.head.i - token.i
+        else:
+            dep_distance = 0
+
+        if not token.is_punct:
             n_words += 1
             total_word_length += len(token.text)
             n_syllables += sum(1 for char in token.text if char in config.VOWELS)
@@ -138,6 +159,7 @@ def scream_metrics(doc: Doc) -> Doc:
             swevoc_entry = lookup(token.text.lower(), token.pos_)
 
             if swevoc_entry:
+
                 swevoc_dict["total"] += 1
                 for swevoc_category in swevoc_entry:
                     swevoc_dict[swevoc_category] += 1
@@ -149,8 +171,6 @@ def scream_metrics(doc: Doc) -> Doc:
             n_content_words += token.pos_ in config.CONTENT_WORD_TAGS
             n_verbs += token.pos_ == config.VERB_TAG
 
-            dep_distance = token.head.i - token.i
-
             match token.dep_:
                 case config.SUBCLAUSE_DEP:
                     n_sub_clauses += 1
@@ -169,18 +189,24 @@ def scream_metrics(doc: Doc) -> Doc:
                 case config.ROOT:
                     sentence_depth = get_sentence_depth(token)
                     if token.pos_ == config.VERB_TAG:
-                        n_verbial_root += 1
+                        n_verbal_roots += 1
 
             if dep_distance > 0:
                 n_right_dependencies += 1
 
-            total_dep_distance += abs(dep_distance)
+        total_dep_distance += abs(dep_distance)
+
+    for key in config.UPOS_POS_TYPES:
+        pos_ud_probs[key] = calculate_safe_division(config.UPOS_POS_TYPES[key], n_words)
+
+    for key in config.DEP_UD_TYPES:
+        dep_ud_probs[key] = calculate_safe_division(config.DEP_UD_TYPES[key], n_words)
 
     # ----- Sentence metrics -----
 
     n_sentences = 0
-    sentence_lengths = 0  # don't count punctuations
-    sentence_length_list_np = np.zeros(len(doc_sents))  # preallocate array
+    sentence_lengths = 0
+    sentence_length_list_np = np.zeros(len(doc_sents))  # preallocate np array
     verb_arities = []
 
     verb_arities_dict = {
@@ -266,6 +292,7 @@ def scream_metrics(doc: Doc) -> Doc:
         "avg_sentence_length": calculate_safe_division(sentence_lengths, n_sentences),
         "avg_word_length": calculate_safe_division(total_word_length, n_words),
         "avg_n_syllables": calculate_safe_division(n_syllables, n_words),
+        "n_words": n_words,
     }
 
     additional_surface_metrics = {
@@ -302,6 +329,7 @@ def scream_metrics(doc: Doc) -> Doc:
         "n_swevoc_h": swevoc_dict["H"],
         "swevoc_s_ratio": calculate_safe_division(swevoc_dict["S"], n_words),
         "swevoc_k_ratio": calculate_safe_division(swevoc_dict["K"], n_words),
+        "swevoc_dict": swevoc_dict,
     }
 
     # ===== Structural vars =====
@@ -320,11 +348,16 @@ def scream_metrics(doc: Doc) -> Doc:
         "avg_nominal_postmodifiers": calculate_safe_division(
             n_nominal_postmodifiers, n_sentences
         ),
-        "avg_nominal_prep_comp": calculate_safe_division(n_prep_comp, n_sentences),
+        "avg_prep_comp": calculate_safe_division(n_prep_comp, n_sentences),
         "avg_sentence_depth": calculate_safe_division(sentence_depth, n_sentences),
-        "verbial_root_ratio": calculate_safe_division(n_verbial_root, n_verbs),
+        "verbial_root_ratio": calculate_safe_division(n_verbal_roots, n_verbs),
         "avg_verb_arity": calculate_safe_division(total_verb_arity, n_verbs),
         "verb_arities_unigram_probs": verb_arity_unigram_probs,
+        "avg_words_per_clause": calculate_safe_division(
+            n_words, n_sentences + n_sub_clauses
+        ),
+        "dep_ud_probs": dep_ud_probs,
+        "pos_ud_probs": pos_ud_probs,
         "lexical_density": calculate_safe_division(n_content_words, n_words),
     }
 
@@ -332,14 +365,11 @@ def scream_metrics(doc: Doc) -> Doc:
         "n_verbs": n_verbs,
         "total_verb_arity": total_verb_arity,
         "total_dep_distance": total_dep_distance,
-        "n_verbial_root": n_verbial_root,
+        "n_verbal_roots": n_verbal_roots,
         "verb_arities_dict": verb_arities_dict,
+        "n_prep_comp": n_prep_comp,
     }
 
-    pos_counts = doc.count_by(spacy.attrs.POS)
-
-    dep_counts = sum(values for _, values in doc.count_by(spacy.attrs.DEP).items())
-
     # ===== Add metrics to doc object =====
 
     additional_metrics = {