Merge branch '__rultor'

models/README.md

@@ -37,19 +37,9 @@ You will need [Docker] installed.
 
 ## How to build new dataset?
 
-Dataset used for model training are located here:
-[train.csv](https://github.com/h1alexbel/samples-filter/blob/dataset/train.csv)
-To refresh it, run [srdataset] either on cloud VM or locally. The building
-process can take a while. After it completed, you should have `dataset.csv`
-file with all collected repositories with the following structure:
-
-* `name`: repository full name, e.g. `redisson/redisson-examples`.
-* `readme`: repository README.md file.
-* `description`: repository description.
-* `topics`: a set of repository topics, e.g. `[apache, streaming, kafka]`
-* `CPD`: commits per day calculated metric.
-* `RC`: published releases to commits ratio.
-* `IC`: issues to commits ratio.
+To build a new dataset, run [srdataset] either on cloud VM or locally. The
+building process can take a while. After it completed, you should have
+`repos.csv` file with all collected repositories.
 
 All features must be preprocessed and vectorized using [pipeline.py].
 Once you have vectors, you can [feed](#how-to-train-it) them to the models.

models/model_tests/pre/test_pre_topics.py → models/model/__init__.py

@@ -19,22 +19,3 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
-import unittest
-
-from model.pre.pre_topics import PreTopics
-
-"""
-Test cases for PreTopics.
-"""
-
-
-class TestPreTopics(unittest.TestCase):
-
-    def test_preprocesses_topics(self):
-        tokens = PreTopics(["java", "examples", "flink", "streaming"]).tokens()
-        expected = ["java", "example", "flink", "streaming"]
-        self.assertEqual(
-            tokens,
-            expected,
-            f"received tokens {tokens} do not match with expected {expected}"
-        )

models/model_tests/pre/test_pre_name.py → models/model/pre/__init__.py

@@ -19,23 +19,3 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
-import unittest
-
-from model.pre.pre_name import PreName
-
-"""
-Test cases for PreName.
-"""
-
-
-class TestPreName(unittest.TestCase):
-
-    def test_preprocesses_name(self):
-        input = "streaming-with-flink/examples-java"
-        tokens = PreName(input).tokens()
-        expected = ["streaming", "flink", "example", "java"]
-        self.assertEqual(
-            tokens,
-            expected,
-            f"received tokens {tokens} for input: {input} do not match with expected {expected}"
-        )

models/model/pre/embeddings.py

@@ -19,47 +19,29 @@
 # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
-from transformers import BertTokenizer, BertModel
+from transformers import BertTokenizer, BertModel, AutoTokenizer
 import torch
 
 """
 Generate embeddings for a set of tokens.
 """
 
 
+# text -> numerical representations -> vector
+# 768, defined by the BERT architecture
 class Embeddings:
-    def __init__(self, tokens, length, encoder="bert-base-uncased"):
-        self.tokens = tokens
+    def __init__(self, raw, length, tokenizer):
+        self.raw = raw
         self.length = length
-        self.tokenizer = BertTokenizer.from_pretrained(encoder)
-        self.model = BertModel.from_pretrained(encoder)
+        self.tokenizer = tokenizer
 
     def embed(self):
-        print(f"Generating embeddings for {self.tokens}")
-        print(f"Encoder: {self.tokenizer}, {self.model}, output length: {self.length}")
-        inputs = []
-        masks = []
-        # @todo #143:30min We generate embeddings for each token instead of the whole unit.
-        #   For now, we generate embeddings for each token. We probably should
-        #   generate embeddings for joined tokens as one unit. In this case we
-        #   can try to replace preprocessing steps with a huggingface tokenizers.
-        #   Let's validate this assumption.
-        for tokens in self.tokens:
-            ids = self.tokenizer.encode_plus(
-                tokens,
-                add_special_tokens=True,
-                return_tensors='pt',
-                padding='max_length',
-                truncation=True,
-                max_length=self.length
-            )
-            inputs.append(ids["input_ids"])
-            masks.append(ids["attention_mask"])
-        inputs = torch.cat(inputs, dim=0)
-        masks = torch.cat(masks, dim=0)
-        with torch.no_grad():
-            outputs = self.model(inputs, attention_mask=masks)
-            states = outputs.last_hidden_state
-        embeddings = states[0].numpy()
-        print(f"Generated embeddings {embeddings}")
-        return embeddings
+        tokens = self.tokenizer.tokenize(
+            self.raw,
+            padding=True,
+            truncation=True,
+            return_tensors='pt'
+        )
+        ids = self.tokenizer.convert_tokens_to_ids(tokens)
+        final = self.tokenizer.prepare_for_model(ids)
+        return final

models/model/pre/pipeline.py

@@ -21,10 +21,6 @@
 # SOFTWARE.
 
 from model.pre.embeddings import Embeddings
-from model.pre.pre_description import PreDescription
-from model.pre.pre_name import PreName
-from model.pre.pre_readme import PreReadme
-from model.pre.pre_topics import PreTopics
 from model.pre.vector import Vector
 
 """
@@ -39,25 +35,28 @@ class Pipeline:
     :param repository Repository to vectorize
     """
 
-    def __init__(self, repository):
+    def __init__(self, repository, tokenizer):
         self.repository = repository
+        self.tokenizer = tokenizer
 
     def apply(self):
         name = self.repository["name"]
         print(f"processing {name}")
-        name = PreName(name).tokens()
-        readme = PreReadme(self.repository["readme"]).tokens()
-        description = PreDescription(self.repository["description"]).tokens()
-        topics = PreTopics(self.repository["topics"]).tokens()
-        e_name = Embeddings(name, 30).embed()
-        e_readme = Embeddings(readme, 512).embed()
-        e_description = Embeddings(description, 100).embed()
-        e_topics = Embeddings(topics, 100).embed()
         return Vector(
-            e_name,
-            e_readme,
-            e_description,
-            e_topics,
+            Embeddings(name, 30, self.tokenizer).embed()["input_ids"],
+            Embeddings(
+                self.repository["readme"],
+                512,
+                self.tokenizer
+            ).embed()["input_ids"],
+            Embeddings(
+                self.repository["description"],
+                100,
+                self.tokenizer
+            ).embed()["input_ids"],
+            Embeddings(
+                self.repository["topics"], 100, self.tokenizer
+            ).embed()["input_ids"],
             cpd=self.repository["cpd"],
             rc=self.repository["rc"],
             ic=self.repository["ic"]