factorExtractor.py

import queue
import re
import threading
import time
from collections import Counter
from typing import Dict, List, Tuple

import pandas as pd
from sqlalchemy import create_engine

import GlobalConstants
from ConfigOperator import ConfigOperator
from FileOperator import FileOperator
from models.RepoInfo import RepoInfo
from MySQLOperator import MySQLOperator

# 获取到的必须是该克隆对产生时候的信息。

# with open("C:/Users/Administrator/getProjectsFromGithub/test/config.yml", "r") as f:
#     config = yaml.load(f, Loader=yaml.FullLoader)
#
# headers = {
#     "User-Agent": "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3941.4 Safari/537.36",
#     "Authorization": "token " + config.get("github_token"),
#     "Content-Type": "application/json",
#     "method": "GET",
#     "Accept": "application/vnd.github.squirrel-girl-preview+json",
# }


# def project_factor_extract(
#         funciton_id_1: int, function_id_2: int, repoInfo: RepoInfo, mysqlOp: MySQLOperator
# ):
#     # if we have the json file when we get all the repos:
#     # with open(path, "r") as f:
#     #     row_data = json.load(f)
#     # stars = []
#     # watchers = []
#     # forks = []
#     # lifetime = []
#     # for d in row_data:
#     #     stars.append(d.get("stargazers_count"))
#     #     watchers.append(d.get("watchers_count"))
#     #     forks.append(d.get("forks_count"))
#     #     life = d.get("updated_at") + " - " + d.get("created_at")
#     #     lifetime.append(life)
#
#     ownername = repoInfo.ownername
#     reponame = repoInfo.reponame
#     url = "https://api.github.com/search/repositories?q=repo:{ownername}/{reponame}".format(
#         ownername=ownername,
#         reponame=reponame,
#     )
#     response_project = requests.get(url=url, headers=headers)
#     response_project.encoding = "utf-8"
#     result = json.loads(response_project.text)
#     projectInfo = result["items"][0]
#     lifetime = projectInfo.get("updated_at") + " - " + projectInfo.get("created_at")
#     stars = projectInfo.get("stargazers_count")
#     forks = projectInfo.get("forks_count")
#     watchers = projectInfo.get("watchers_count")
#     contributor_url = projectInfo.get("contributors_url")
#     response_con = requests.get(url=contributor_url, headers=headers)
#     response_con.encoding = "utf-8"
#     contributors = len(json.loads(response_con.text))
#
#     table_commits = mysqlOp.tablename_dict["commits"]
#     sql_commits = "select count(*) from `{tablename}`".format(
#         tablename=table_commits,
#     )
#     mysqlOp.cursor.execute(sql_commits)
#     commits = mysqlOp.cursor.fetchone().get("count(*)")
#     return lifetime, stars, forks, watchers, contributors, commits


def factor_extractor(function_id_1: int, function_id_2: int, mysqlOp: MySQLOperator):
    # TODO:1.get lifetime,stars,forks,watchers,commits and authors of project when the clone pair exist!
    #      2.get the similarity,degree diff,is_test,file_distance,same_function_name(bool),history_change_sum,co-change(bool),consistant_change(bool)
    method_function_relations = mysqlOp.tablename_dict["method_function_relations"]
    clone_relations_function = mysqlOp.tablename_dict["clone_relations_function"]
    commits = mysqlOp.tablename_dict["commits"]
    filepaths = mysqlOp.tablename_dict["filepaths"]
    blob_methods = mysqlOp.tablename_dict["blob_methods"]
    blob_commit_relations = mysqlOp.tablename_dict["blob_commit_relations"]
    commit_relations = mysqlOp.tablename_dict["commit_relations"]

    relate_commits = []
    sql_clone_pairs = """
                select commit_id from `{tablename}`
                where (function_id_1 = {function_id_1} and function_id_2 = {function_id_2}) or
                (function_id_1 = {function_id_2} and function_id_2 = {function_id_1})
            """.format(
        tablename=clone_relations_function,
        function_id_1=function_id_1,
        function_id_2=function_id_2,
    )
    mysqlOp.cursor.execute(sql_clone_pairs)
    clone_pairs = mysqlOp.cursor.fetchall()
    for clone_pair in clone_pairs:
        relate_commits.append(clone_pair.get("commit_id"))
    # get the related commmits of clone pair
    candidate_commits = [
        commit_id for commit_id in range(min(relate_commits), max(relate_commits) + 1)
    ]

    start_commits = []
    end_commits = []
    middle_commits = []

    commit_children_dict = {}  # find children by parent_id
    commit_parents_dict = {}  # find parents by child_id
    mysqlOp.cursor.execute(
        "select id, parent_id from `{commit_relations}`".format(
            commit_relations=commit_relations
        )
    )
    commit_relation_results = mysqlOp.cursor.fetchall()
    for relation in commit_relation_results:
        commit_id = relation["id"]
        parent_id = relation["parent_id"]
        if commit_id is not None:
            commit_parents_dict.setdefault(commit_id, [])
        if parent_id is not None:
            commit_parents_dict.setdefault(parent_id, [])
        if commit_id is not None:
            commit_children_dict.setdefault(commit_id, [])
        if parent_id is not None:
            commit_children_dict.setdefault(parent_id, [])
        if parent_id is not None:
            commit_parents_dict[commit_id].append(parent_id)
            commit_children_dict[parent_id].append(commit_id)

    # if the clone pair only live in one commit, this commit belongs to both start commit and end commit
    for commit_id in candidate_commits:
        if commit_id not in commit_parents_dict:
            parent_ids = []
        else:
            parent_ids = commit_parents_dict[commit_id]

        if commit_id not in commit_children_dict:
            son_ids = []
        else:
            son_ids = commit_children_dict[commit_id]

        intersect_parents = list(set(parent_ids) & set(candidate_commits))
        intersect_children = list(set(son_ids) & set(candidate_commits))
        # if no parent in candidate_commits & at least one child in candidate_commits & there exists clone relationship in this commit -> candidate_start
        if (
            len(intersect_parents) == 0
            and len(intersect_children) > 0
            and commit_id in relate_commits
        ):
            start_commits.append(commit_id)
        # if at least one parent in candidate_commits & no child in candidate_commits & there exists clone relationship in this commit -> candidate_end
        elif (
            len(intersect_parents) > 0
            and len(intersect_children) == 0
            and commit_id in relate_commits
        ):
            end_commits.append(commit_id)
        # if no parent in candidate_commits & no child in candidate_commits -> ignore
        elif (
            len(intersect_parents) == 0
            and len(intersect_children) == 0
            and commit_id not in relate_commits
        ):
            continue
        elif (
            len(intersect_parents) == 0
            and len(intersect_children) == 0
            and commit_id in relate_commits
        ):
            start_commits.append(candidate_commits[0])
            end_commits.append(candidate_commits[0])
        # if at least one parent in candidate_commits & at least one child in candidate_commits -> middle_commit
        elif len(intersect_parents) > 0 and len(intersect_children) > 0:
            middle_commits.append(commit_id)

    begin_time = time.time()
    for start_commit_id in start_commits:
        sql_clone_begin = """
            select id, sha, commit_time from `{tablename}` where id = {commit_id}
        """.format(
            tablename=commits,
            commit_id=start_commit_id,
        )
        mysqlOp.cursor.execute(sql_clone_begin)
        begin_commit = mysqlOp.cursor.fetchone()
        if begin_commit.get("commit_time") <= begin_time:
            begin_commit_id = begin_commit.get("id")
            # begin_commit_sha = begin_commit.get("sha")
            begin_time = begin_commit.get("commit_time")

    # TODO:similarity
    sql_similarity = """
         select similarity from `{tablename}` where commit_id={commit_id} and (
         (function_id_1={function_id_1} and function_id_2={function_id_2}) or
         (function_id_1={function_id_2} and function_id_2={function_id_1})
         )
    """.format(
        tablename=clone_relations_function,
        function_id_1=function_id_1,
        function_id_2=function_id_2,
        commit_id=begin_commit_id,
    )
    mysqlOp.cursor.execute(sql_similarity)
    similarity = mysqlOp.cursor.fetchone().get("similarity")

    # TODO:degree_diff
    sql_degree_1 = """
        select distinct function_id_1,function_id_2 from `{tablename}` where commit_id={commit_id} and (
        function_id_1={function_id_1} or function_id_2={function_id_1}
        )
    """.format(
        tablename=clone_relations_function,
        function_id_1=function_id_1,
        commit_id=begin_commit_id,
    )
    mysqlOp.cursor.execute(sql_degree_1)
    points_1 = []
    pairs = mysqlOp.cursor.fetchall()
    for pair in pairs:
        points_1.append(pair.get("function_id_1"))
        points_1.append(pair.get("function_id_2"))
    degree_1 = len(Counter(points_1))
    sql_degree_2 = """
        select distinct function_id_1,function_id_2 from `{tablename}` where commit_id={commit_id} and (
        function_id_1={function_id_2} or function_id_2={function_id_2}
        )
    """.format(
        tablename=clone_relations_function,
        function_id_2=function_id_2,
        commit_id=begin_commit_id,
    )
    mysqlOp.cursor.execute(sql_degree_2)
    points_2 = []
    pairs = mysqlOp.cursor.fetchall()
    for pair in pairs:
        points_2.append(pair.get("function_id_1"))
        points_2.append(pair.get("function_id_2"))
    degree_2 = len(Counter(points_2))
    degree_diff = degree_1 - degree_2

    # TODO:is_test and file_distance
    sql_first = """
        select blob_id_1, blob_id_2, method_id_1, method_id_2 from `{tablename}`
        where commit_id = {commit_id} and (
        (function_id_1={function_id_1} and function_id_2={function_id_2}) or
        (function_id_1={function_id_2} and function_id_2={function_id_1})
        )
    """.format(
        tablename=clone_relations_function,
        function_id_1=function_id_1,
        function_id_2=function_id_2,
        commit_id=begin_commit_id,
    )
    mysqlOp.cursor.execute(sql_first)
    first_pair = mysqlOp.cursor.fetchone()
    sql_filepath = """
        select filepath from `{tablename1}` t1, `{tablename2}` t2 where t1.filepath_id = t2.id
        and commit_id = {commit_id} and (blob_id = {blob_id_1} or blob_id = {blob_id_2})
    """.format(
        tablename1=blob_commit_relations,
        tablename2=filepaths,
        commit_id=begin_commit_id,
        blob_id_1=first_pair.get("blob_id_1"),
        blob_id_2=first_pair.get("blob_id_2"),
    )
    mysqlOp.cursor.execute(sql_filepath)
    filepath = mysqlOp.cursor.fetchall()
    # filepath = [{'filepath': b'src/test/Test1.java'},{'filepath': b'src\\Test1.java'}]
    # file_distance定义参考ccfinder文章：克隆所在文件的除公共文件路径外的最长文件路径长度
    try:
        if len(filepath) == 1:
            file_distance = 0
            filepath = filepath[0].get("filepath").decode()
            path = re.split("[\\\\ |/]", filepath)
            if "test" or "Test" in path:
                is_test = 2
        else:
            if filepath[0] == filepath[1]:
                file_distance = 0
                filepath = filepath[0].get("filepath").decode()
                path = re.split("[\\\\ |/]", filepath)
                if "test" or "Test" in path:
                    is_test = 2
            else:
                is_test = 0
                filepath_1 = filepath[0].get("filepath").decode()
                filepath_2 = filepath[1].get("filepath").decode()
                while len(filepath_1) > 1:
                    path_1 = re.split("[\\\\ |/]", filepath_1, 1)
                    path_2 = re.split("[\\\\ |/]", filepath_2, 1)
                    if (path_1[0] == "test" or path_1[0] == "Test") and is_test <= 2:
                        is_test = is_test + 1
                    if (path_2[0] == "test" or path_2[0] == "Test") and is_test <= 2:
                        is_test = is_test + 1
                    if path_1[0] == path_2[0]:
                        file_distance = 0
                        break
                        del path_1[0]
                        del path_2[0]
                        filepath_1 = path_1[0]
                        filepath_2 = path_2[0]
                    else:
                        if len(path_1) > len(path_2):
                            file_distance = len(path_1)
                        else:
                            file_distance = len(path_2)
                        break

    except Exception:
        print(repoInfo.repo_id)
        print(function_id_1, function_id_2)
        print(filepath)

    # TODO:function_name_same
    sql_name_1 = """
        select function_name from `{tablename}` where function_id={function_id}
    """.format(
        tablename=blob_methods,
        function_id=function_id_1,
    )
    mysqlOp.cursor.execute(sql_name_1)
    function_name_1 = mysqlOp.cursor.fetchone().get("function_name")
    sql_name_2 = """
        select function_name from `{tablename}` where function_id={function_id}
    """.format(
        tablename=blob_methods,
        function_id=function_id_2,
    )
    mysqlOp.cursor.execute(sql_name_2)
    function_name_2 = mysqlOp.cursor.fetchone().get("function_name")
    function_name_same = function_name_1 == function_name_2

    # TODO:history_change_sum
    sql_history_change_1 = """
        with recursive tmp as(
            select * from `{tablename}` where method_id_2={method_id}
            UNION distinct
            select t2.* from `{tablename}` t2 inner join tmp t1 on t1.method_id_1=t2.method_id_2
        )select method_id_1,method_id_2 from tmp
    """.format(
        tablename=method_function_relations, method_id=first_pair.get("method_id_1")
    )
    mysqlOp.cursor.execute(sql_history_change_1)
    changes_1 = mysqlOp.cursor.fetchall()
    methods_1 = []
    for change in changes_1:
        methods_1.append(change.get("method_id_2"))
    history_change_1 = []
    for i in range(len(methods_1)):
        mysqlOp.cursor.execute(
            "select commit_id from `{tablename1}` bm, `{tablename2}` bcr where bm.id={method_id} and bm.blob_id = bcr.blob_id".format(
                tablename1=blob_methods,
                tablename2=blob_commit_relations,
                method_id=methods_1[i],
            )
        )
        commit_ids_1 = mysqlOp.cursor.fetchall()
        for commit_id in commit_ids_1:
            mysqlOp.cursor.execute(
                "select commit_time from `{tablename}` where id={commit_id}".format(
                    tablename=commits,
                    commit_id=commit_id.get("commit_id"),
                )
            )
            commit_time = mysqlOp.cursor.fetchone().get("commit_time")
            if commit_time <= begin_time:
                history_change_1.append(changes_1)

    sql_history_change_2 = """
        with recursive tmp as(
            select * from `{tablename}` where method_id_2={method_id}
            UNION distinct
            select t2.* from `{tablename}` t2 inner join tmp t1 on t2.method_id_2=t1.method_id_1
        )select method_id_1,method_id_2 from tmp
    """.format(
        tablename=method_function_relations, method_id=first_pair.get("method_id_2")
    )
    mysqlOp.cursor.execute(sql_history_change_2)
    changes_2 = mysqlOp.cursor.fetchall()
    methods_2 = []
    for change in changes_2:
        methods_2.append(change.get("method_id_2"))
    history_change_2 = []
    for j in range(len(methods_2)):
        mysqlOp.cursor.execute(
            "select commit_id from `{tablename1}` bm, `{tablename2}` bcr where bm.id={method_id} and bm.blob_id = bcr.blob_id".format(
                tablename1=blob_methods,
                tablename2=blob_commit_relations,
                method_id=methods_2[j],
            )
        )
        commit_ids_2 = mysqlOp.cursor.fetchall()
        for commit_id in commit_ids_2:
            mysqlOp.cursor.execute(
                "select commit_time from `{tablename}` where id={commit_id}".format(
                    tablename=commits,
                    commit_id=commit_id.get("commit_id"),
                )
            )
            commit_time = mysqlOp.cursor.fetchone().get("commit_time")
            if commit_time <= begin_time:
                history_change_2.append(changes_2)

    history_change_sum = len(history_change_1) + len(history_change_2)
    # TODO:co-change and consistant-change
    sql_parent_commit = """
        select parent_id from `{tablename}` where id={commit_id}
    """.format(
        tablename=commit_relations,
        commit_id=begin_commit_id,
    )
    mysqlOp.cursor.execute(sql_parent_commit)
    parents = mysqlOp.cursor.fetchall()
    parent_ids = []
    for parent in parents:
        parent_ids.append(parent.get("parent_id"))
    related_methods = []
    related_method_id = []
    for id in parent_ids:
        if id is None:
            id = 0
        sql = """
            select bm.id as method_id from `{tablename1}` bm, `{tablename2}` bcr where bcr.commit_id = {commit_id} and bcr.blob_id = bm.blob_id
        """.format(
            tablename1=blob_methods,
            tablename2=blob_commit_relations,
            commit_id=id,
        )
        mysqlOp.cursor.execute(sql)
        related_methods.extend(mysqlOp.cursor.fetchall())
    for related_method in related_methods:
        related_method_id.append(related_method.get("method_id"))

    target_change_1 = {}
    sql_change_method_1 = """
        select * from `{tablename}` where method_id_2 = {method_id}
    """.format(
        tablename=method_function_relations, method_id=first_pair.get("method_id_1")
    )
    mysqlOp.cursor.execute(sql_change_method_1)
    change_method_1 = mysqlOp.cursor.fetchall()
    change_method_id_1 = []
    for change_method in change_method_1:
        change_method_id_1.append(change_method.get("method_id_1"))
    for i in range(len(change_method_id_1)):
        if change_method_id_1[i] in related_method_id:
            target_change_1 = change_method_1[i]
            break

    target_change_2 = {}
    sql_change_method_2 = """
            select * from `{tablename}` where method_id_2 = {method_id}
        """.format(
        tablename=method_function_relations, method_id=first_pair.get("method_id_2")
    )
    mysqlOp.cursor.execute(sql_change_method_2)
    change_method_2 = mysqlOp.cursor.fetchall()
    change_method_id_2 = []
    for change_method in change_method_2:
        change_method_id_2.append(change_method.get("method_id_2"))
    for i in range(len(change_method_id_2)):
        if change_method_id_2[i] in related_method_id:
            target_change_2 = change_method_2[i]
            break

    if target_change_1 == {} or target_change_2 == {}:
        co_change = 0
        consistant_change = 0
    else:
        co_change = 1
        if (
            target_change_1.get("change") == target_change_2.get("change")
            and target_change_1.get("change") is not None
            and target_change_2.get("change") is not None
        ):
            consistant_change = 1
        else:
            consistant_change = 0

    # TODO:main_author_same
    sql_function_1_author = """
        select author_name,min(commit_time) from `{tablename1}` c ,`{tablename2}` bcr,`{tablename3}` bm
        where bm.function_id = {function_id} and bm.blob_id = bcr.blob_id and bcr.commit_id = c.id
        group by author_name
    """.format(
        tablename1=commits,
        tablename2=blob_commit_relations,
        tablename3=blob_methods,
        function_id=function_id_1,
    )
    mysqlOp.cursor.execute(sql_function_1_author)
    author_1 = mysqlOp.cursor.fetchall()[0].get("author_name").decode()

    sql_function_2_author = """
            select author_name,min(commit_time) from `{tablename1}` c ,`{tablename2}` bcr,`{tablename3}` bm
            where bm.function_id = {function_id} and bm.blob_id = bcr.blob_id and bcr.commit_id = c.id
            group by author_name
        """.format(
        tablename1=commits,
        tablename2=blob_commit_relations,
        tablename3=blob_methods,
        function_id=function_id_2,
    )
    mysqlOp.cursor.execute(sql_function_2_author)
    author_2 = mysqlOp.cursor.fetchall()[0].get("author_name").decode()

    main_author_same = author_1 == author_2

    # TODO:author_exp_sum
    sql_exp = """
        select count(author_name) from `{tablename}` where (author_name = "{author1}" or author_name = "{author2}") and commit_time <= {time}
    """.format(
        tablename=commits, author1=author_1, author2=author_2, time=begin_time
    )
    mysqlOp.cursor.execute(sql_exp)
    author_exp_sum = mysqlOp.cursor.fetchone().get("count(author_name)")

    return {
        "function_id_1": function_id_1,
        "function_id_2": function_id_2,
        "similarity": similarity,
        "degree_diff": degree_diff,
        "is_test": is_test,
        "file_distance": file_distance,
        "function_name_same": function_name_same,
        "history_change_sum": history_change_sum,
        "co_change": co_change,
        "consistant_change": consistant_change,
        "main_author_same": main_author_same,
        "author_exp_sum": author_exp_sum,
    }


def run(q):
    while not q.empty():
        repoInfo: RepoInfo = q.get()
        mysqlOp: MySQLOperator = MySQLOperator(
            config_path="config.yml", autocommit=True, repoInfo=repoInfo
        )
        clone_relations_function = mysqlOp.tablename_dict["clone_relations_function"]
        sql_all_clones = """
                            select distinct function_id_1,function_id_2 from `{tablename}`
                        """.format(
            tablename=clone_relations_function
        )
        while True:
            try:
                mysqlOp.cursor.execute(sql_all_clones)
                all_clone_pairs = mysqlOp.cursor.fetchall()
                break
            except Exception:
                mysqlOp.connection.ping(reconnect=True)
        factor_list = []
        for clone_pair in all_clone_pairs:
            function_id_1 = clone_pair.get("function_id_1")
            function_id_2 = clone_pair.get("function_id_2")
            factor_dict = factor_extractor(function_id_1, function_id_2, mysqlOp)
            factor_list.append(factor_dict)
        result = pd.DataFrame(factor_list)
        factors = "{repo_id}{separator}factors".format(
            repo_id=repoInfo.repo_id, separator=GlobalConstants.SEPARATOR
        )
        sql_result = """
                            create table if not exists `{tablename}` (
                                `id` int(11) NOT NULL AUTO_INCREMENT,
                                `function_id_1` int(11) NULL,
                                `function_id_2` int(11) NULL,
                                `similarity` int(11) NULL,
                                `degree_diff` int(11) NULL,
                                `is_test` int(11) NULL,
                                `file_distance` int(11) NULL,
                                `function_name_same` int(11) NULL,
                                `history_change_sum` int(11) NULL,
                                `co_change` int(11) NULL,
                                `consistant_change` int(11) NULL,
                                `main_author_same` int(11) NULL,
                                `author_exp_sum` int(11) NULL,
                                PRIMARY KEY (`id`),
                                INDEX(`function_id_1`) USING BTREE,
                                INDEX(`function_id_2`) USING BTREE,
                                INDEX(`similarity`) USING BTREE,
                                INDEX(`degree_diff`) USING BTREE,
                                INDEX(`is_test`) USING BTREE,
                                INDEX(`file_distance`) USING BTREE,
                                INDEX(`function_name_same`) USING BTREE,
                                INDEX(`history_change_sum`) USING BTREE,
                                INDEX(`co_change`) USING BTREE,
                                INDEX(`consistant_change`) USING BTREE,
                                INDEX(`main_author_same`) USING BTREE,
                                INDEX(`author_exp_sum`) USING BTREE
                            )
                    """.format(
            tablename=factors
        )
        mysqlOp.cursor.execute(sql_result)
        mysqlOp.truncate_table(tablename=factors)
        config = mysqlOp.config["mysql"]
        engine = create_engine(
            "mysql+pymysql://{username}:{password}@{host}:{port}/{database}".format(
                username=config["user"],
                password=config["passwd"],
                host=config["host"],
                port=config["port"],
                database=config["database"],
            )
        )
        result.to_sql(
            factors,
            engine,
            index=False,
            if_exists="append",
        )
        q.task_done()
        print(
            "finish extract factors in repo: {git_url}".format(git_url=repoInfo.git_url)
        )


if __name__ == "__main__":
    repoInfos: List[RepoInfo] = FileOperator("factor_repos").load_repos()
    mysqlOp = MySQLOperator(config_path=GlobalConstants.CONFIG_PATH)
    workQueue = queue.Queue()
    for repoInfo in repoInfos:
        # query the id of this repository
        repo_id = mysqlOp.get_repo_id_by_names(repoInfo=repoInfo)
        repoInfo.repo_id = repo_id
        workQueue.put(repoInfo)
    threads = []
    for i in range(
        int(
            ConfigOperator(config_path=GlobalConstants.CONFIG_PATH).read_config()[
                "RCD"
            ]["thread_num"]
        )
    ):
        t = threading.Thread(target=run, args=(workQueue,))
        t.start()
        threads.append(t)
    for t in threads:
        t.join()
    print("Finish")