exportLogData.py

# Module to export the collaboration network to different SNA tool formats 
# First implementation to uciNet for WebKit SNA ACM SIG MIS paper - on this file 
# Second implementation to GraphML (VISIONE) for OpenStack SNA - Open Journal special issue  - using the exportgraphml form


from __future__ import absolute_import
from __future__ import print_function
import sys
import re
import csv
from datetime import * 
import networkx as nx
import exportGraphMLformat

# Replace '@' and '.' by "AT" and "DOT" 
def clearDotsAndAts(contribEmail):

    #print ("clearing Strings From Dots and Ats from: [" + contribEmail+ "]")

    pattern = re.compile('([\w\-\.]+@(\w[\w\-]+\.)+[\w\-]+)')
 
    if (pattern.search(contribEmail)== None):
        print ("ERROR Contributor have an invalidName")
        exit()
    
    tmp= re.sub('\.','DOT' ,contribEmail)
    tmp= re.sub('\@','AT' ,tmp)

    return tmp

# Create a Network file (uciNet style) in raw text 
def createNetworkFile(tuplesList , outFileName):
    
    print ("")    
    print(("Writing network on file:[" + outFileName + "]"))

    f = open(outFileName, 'w')

    f.write('#File generated by scrapLog.py for research purposes \n')
    f.write('#Network connections by scrapping the changelog [' + sys.argv[1] + '] on ' + str(datetime.now()) + '\n')

    for connection in tuplesList:
        for author in connection[0]:
            f.write(clearDotsAndAts(author) + "\t")
        f.write('\n')

    f.write('END \n')
      

# Create a Atributes file (uciNet style) in raw text 
def createAtributesFile(logData , outFileName):
    
    print ("")    
    print(("Writing atributes on file:[" + outFileName + "]"))

    f = open(outFileName, 'w')

    f.write('#File generated by scrapLog.py for research purposes \n')
    f.write('#Node atributes by scrapping the changelog [' + sys.argv[1] + '] on ' + str(datetime.now()) + '\n')

    f.write('NAME\tAFFILIATION\n')

    for change in logData:
        f.write(clearDotsAndAts(change[0][1]))
        f.write('\t')
        f.write(change[0][2])
        f.write('\n')

    f.write('END \n')
      

# Create a Network file (uciNet style) in CSV file for spreedsheet software
def createNetworkFileCSV(tuplesList , outFileName):
    
    print ("")    
    print(("Writing network on file (.CSV):[" + outFileName + "]"))


    csvfile = open(outFileName, 'w')
    #csvfile = open(outFileName, 'wb')
    

    atribWriter = csv.writer(csvfile, dialect='excel', delimiter=' ') 
    
    # Print headers
    #csvfile.write('#File generated by scrapLog.py for research purposes \n')
    #csvfile.write('#Network connections by scrapping the changelog [' + sys.argv[1] + '] on ' + str(datetime.now()) + '\n')

    atribWriter.writerow(["NODE", "NODE"])
    for connection in tuplesList:
        tmp= []
        for author in connection[0]:
            tmp.append(clearDotsAndAts(author))

        atribWriter.writerow(tmp)


    csvfile.close()

    # Success 
    print((str(len(tuplesList)) +" network relationships writen down :[" + outFileName + "]"))

# Create a Atributes file (uciNet style) in CSV file for spreadsheat software
def createAtributesFileCSV(logData , outFileName):
    
    print ("")    
    print(("Writing atributes on file (.CSV):[" + outFileName + "]"))



    csvfile= open(outFileName, 'w')
    #csvfile= open(outFileName, 'wb')
    atribWriter = csv.writer(csvfile, dialect='excel',delimiter=' ')    
    
    # Print option file headers
    #atribWriter.writerow(['#File generated by scrapLog.py for research purposes'])
    #atribWriter.writerow(['#Node atributes by scrapping the changelog ['+ sys.argv[1] + '] on ' + str(datetime.now())])

    atribWriter.writerow(['NAME','AFFILIATION'])

    for change in logData:
        atribWriter.writerow([clearDotsAndAts(change[0][1]), change[0][2]])

    csvfile.close()       

    # Success 
    print((str(len(logData)) +" node atributes writen down :[" + outFileName + "]"))



############### By core companies #############
# As in http://blog.bitergia.com/2013/02/06/report-on-the-activity-of-companies-in-the-webkit-project/ 
# All coreCompanies + bot + core 

coreCompanies = ['apple', 'google', 'nokia', 'rim', 'igalia', 'intel', 'samsung', 'inf' , 'adobe' , 'torchmobile']

coreCompaniesColor =  []


# Create a Network file  (grouped by core companies) (uciNet style) in CSV file for spreedsheet software
def createNetworkByCoreCompaniesFileCSV(tuplesList , outFileName):
    print ("")    
    print(("Writing network (grouped by core companies) on file (.CSV):[" + outFileName + "]"))


    csvfile = open(outFileName, 'w')
    #csvfile = open(outFileName, 'wb')
    

    atribWriter = csv.writer(csvfile, dialect='excel', delimiter=' ') 
    atribWriter.writerow(["NODE", "NODE"])

    # Its equal right ? Just the the atributes change
    for connection in tuplesList:
        tmp= []
        for author in connection[0]:
            tmp.append(clearDotsAndAts(author))

        atribWriter.writerow(tmp)
   
    csvfile.close()



# Create a Atributes, grouped by core companies,  file (uciNet style) in CSV file for spreadsheat software
def createAtributesByCoreFileCSV(logData , outFileName):
    
    print ("")    
    print(("Writing atributes  (grouped by core companies) on file (.CSV):[" + outFileName + "]"))



    csvfile= open(outFileName, 'w')
    #csvfile= open(outFileName, 'wb')
    
    atribWriter = csv.writer(csvfile, dialect='excel',delimiter=' ')    
    
    # Print option file headers

    atribWriter.writerow(['NAME','AFFILIATION'])

    for change in logData:

        email = clearDotsAndAts(change[0][1])
        affiliation = change[0][2] 
        
        if affiliation in coreCompanies:
            atribWriter.writerow([email, affiliation])

        elif email == "webkit.review.bot@gmail.com":
            atribWriter.writerow([email, 'AutomatedBot'])
        elif affiliation not in coreCompanies:
            atribWriter.writerow([email, 'other'])
        else:
            print ("ERROR writing atributes grouped by company")
            exit()

    csvfile.close()


# export the grapth node and edges to GraphML format 
# Must be readable by Visione
def createGraphML(network_with_affiliation_atributes :nx.Graph ,outFileName :str):
    # iterator for nAf  

    print ("")    
    print(("\tExporting graph to file (.graphml):[" + outFileName + "]"))
    
    
    # verify arguments data
    ## verify graph/network 

    
    if network_with_affiliation_atributes.order() < 2:
        print ("\tERROR network have less than two nodes !!")
        exit(1)

    if network_with_affiliation_atributes.size() < 1:
        print ("\tERROR network have less than one edge !!")
        exit(1)

        
    ## verify affiliations 

    "verify every node as affiliation data"


    for node, data in network_with_affiliation_atributes.nodes(data=True):
        if len(data['affiliation']) == 0:
            print ("invalid affiliation atribute")
            print (node)
            print (data['affiliation'])
            exit(1)
        if 'affiliation' not in data.keys():
            print ("affiliation atribute is missing")
            print (node)
            print (data['affiliation'])
            exit(1)



    ## verify outFilename
    if type(outFileName) != str:
        print ("\tERROR outfilename must be a string")
        exit()
        
    if len(outFileName) < 5 :
        print ("\tERROR outfilename must be a long string. More than 5 caracters")
        exit()

    if outFileName[-8:] != ".graphML":
        print ("\tERROR outfilename must finish with .grapthML extenssion")
        exit()

    # open the export file 

    print ("")    
    print(("\tWriting grapthML file  (for VISIONE SNA tool or other ) on file:[" + outFileName + "]"))

    gfile= open(outFileName, 'w')

    
    # open XML headers 
    gfile.writelines(exportGraphMLformat.graphml_header)
    
    # Add grapth atributes 

    gfile.writelines(exportGraphMLformat.setNodeAntributeKey(0,"e-mail","string"))
    gfile.writelines(exportGraphMLformat.setNodeAntributeKey(1,"color","string"))
    gfile.writelines(exportGraphMLformat.setNodeAntributeKey(2,"affiliation","string"))
    

    # Open grapth 
    gfile.writelines(exportGraphMLformat.graph_opener)
    
    # store the nodes id for each email/contributor
    tmpNodeId = {} 


    "for now all colors are turquoise"
    


    print ()
    print ("\t\tWriting nodes in graphML file")
    node_id = 0
    for node, data in network_with_affiliation_atributes.nodes(data=True):
        email=node
        afl=data['affiliation']
        #print(exportGraphml.addNode(nAf,[(0,email),(1,"turquoise"),(2,afl)]))
        gfile.writelines(exportGraphMLformat.addNode(node_id,[(0,email),(1,"turquoise"),(2,afl)]))
        # Give a each node and numeric id atribute data as well 
        network_with_affiliation_atributes.nodes[node]['id']= node_id
        node_id+=1


    print ()
    print ("\t\tWriting edges in graphML file")
    
    nTup=0
    for edge in network_with_affiliation_atributes.edges():
        nodeIdFrom = network_with_affiliation_atributes.nodes[edge[0]]['id']
        nodeIdTo = network_with_affiliation_atributes.nodes[edge[1]]['id']
        #print("edge=",edge)
        #print("edge via id=",nodeIdFrom,nodeIdTo)
        
        
        #print(exportGraphml.addEdge("e"+str(nTup),nodeIdFrom,nodeIdTo))
        gfile.writelines(exportGraphMLformat.addEdge("e"+str(nTup),nodeIdFrom,nodeIdTo))
        nTup+=1

        if edge[0] == edge[1]:
            print ("\t ERROR arc between the same mail/node/developer")
            print(("\t edge=["+str(edge)+"]"))
            sys.exit()            
    
    # Close grapth
    gfile.writelines(exportGraphMLformat.graph_closer)

    # close XML document 
    gfile.writelines(exportGraphMLformat.graphml_closer)

    # close the export file 
    gfile.close()