Added host list detections sql upload!

README.md

@@ -33,6 +33,8 @@ hosts = get_host_list(auth, details="All/AGs", show_tags=True, page_count=4)
 
 ## Current Supported Modules 
 
+>**Head's Up!:** SQL DB uploading is currently in development! 🎉
+
 |Module| Status |
 |--|--|
 | GAV (Global AssetView) |✅|

docs/index.md

@@ -28,6 +28,8 @@ hosts = get_host_list(auth, details="All/AGs", show_tags=True, page_count=4)
 >>>[VMDRHost(12345), ...]
 ```
 
+>**Head's Up!:** SQL DB uploading is currently in development! 🎉
+
 ## Current Supported Modules 
 
 |Module| Status |

docs/sql.md

@@ -2,7 +2,7 @@
 
 >**Head's Up!:** ```qualyspy.sql``` is currently in development and has been tested using Microsoft SQL Server only. Other DBs will be tested at some point.
 
-```qualyspy``` supports uploading data it has pulled to a SQL Database using various ```upload_<module>_*``` functions. Thanks to the [Pandas library](https://pandas.pydata.org) and qualyspy's ```BaseList``` class, uploading is rather easy.
+```qualyspy``` supports uploading data it has pulled to a SQL Database using various ```upload_<module>_*``` functions. Thanks to the [Pandas library](https://pandas.pydata.org) and qualyspy's ```BaseList``` class, uploading is rather easy. ```qualyspy``` automatically will create the table for you if it does not exist, and will append data to the table if it does exist. The ```import_datetime``` field is also added to each table to track when the data was uploaded.
 
 ## Steps to Get Going
 
@@ -18,11 +18,24 @@ Next, build your connection object. ```qualyspy``` supports username/password au
 
 ```py
 
-# Get a sqlalchemy.Connection using trusted_connection:
+# Get a sqlalchemy.Connection using trusted_connection to SQL Server.
+# since driver defaults to "ODBC Driver 17 for SQL Server" and db_type defaults to "mssql", you can omit them.
 cnxn = db_connect(host='10.0.0.1', db='qualysdata', trusted_cnxn=True)
 
-# Get a sqlalchemy.Connection with username/password auth:
-cnxn = db_connect(host='10.0.0.1', db='qualysdata', username='Jane', password='SuperSecretPassword!')
+# Get a sqlalchemy.Connection with username/password auth to an oracle DB:
+cnxn = db_connect(host='10.0.0.1', db='qualysdata', username='Jane', password='SuperSecretPassword!', db_type='oracle', driver='Some Driver for Oracle')
+```
+
+Note that you are required to call ```.close()``` on the connection object when you are done with it to close the connection to the DB.
+
+```py
+
+cnxn = db_connect(host='10.0.0.1', db='qualysdata', trusted_cnxn=True)
+
+# Do some stuff with the connection
+...
+
+cnxn.close()
 ```
 
 ### Step 3: Fire Away!
@@ -33,12 +46,13 @@ And finally, you can use the following supported functions:
 
 Each upload function takes 2 parameters. The first is the ```BaseList``` of data, and the second is the ```sqlalchemy.Connection``` object you built above.
 
-| Function Name | Module  | ```qualyspy``` Function Data Source |
-| -- | -- | -- |
-| ```upload_vmdr_ags``` | VMDR | ```vmdr.get_ag_list()```|
-| ```upload_vmdr_kb``` | VMDR | ```vmdr.query_kb()```|
-| ```upload_vmdr_hosts``` | VMDR | ```vmdr.get_host_list()```|
-| ```upload_vmdr_ips``` | VMDR | ```vmdr.get_ip_list()```|
+| Function Name | Module  | ```qualyspy``` Function Data Source | Resulting Table Name |
+| -- | -- | -- | -- |
+| ```upload_vmdr_ags``` | VMDR | ```vmdr.get_ag_list()```| ```vmdr_assetgroups``` |
+| ```upload_vmdr_kb``` | VMDR | ```vmdr.query_kb()```| ```vmdr_knowledgebase``` |
+| ```upload_vmdr_hosts``` | VMDR | ```vmdr.get_host_list()```| ```vmdr_hosts_list``` |
+| ```upload_vmdr_hld``` | VMDR | ```vmdr.get_hld()```| ```vmdr_hld_hosts_list``` for hosts and ```vmdr_hld_detections``` for detections |
+| ```upload_vmdr_ips``` | VMDR | ```vmdr.get_ip_list()```| ```vmdr_ips``` |
 
 ## A Friendly Recommendation For Getting Data
 

qualyspy/sql/__init__.py

@@ -8,4 +8,5 @@
     upload_vmdr_kb,
     upload_vmdr_hosts,
     upload_vmdr_ips,
+    upload_vmdr_hld,
 )

qualyspy/sql/base.py

@@ -65,7 +65,12 @@ def db_connect(
             f"{db_type}+pyodbc://{username}:{password}@{host}/{db}?driver={driver}"
         )
 
-    engine = create_engine(conn_str)
+    if db_type != "mssql" and driver == "ODBC Driver 17 for SQL Server":
+        engine = create_engine(conn_str)
+    else:
+        # We can enable fast_executemany for mssql to speed up inserts:
+        print("Enabling fast_executemany for mssql...")
+        engine = create_engine(conn_str, fast_executemany=True)
 
     return engine.connect()
 
@@ -90,9 +95,9 @@ def upload_data(df: DataFrame, table: str, cnxn: Connection, dtype: dict) -> int
 
     # For any string values in the DataFrame, make sure it doesn't
     # exceed VARCHAR(MAX) length:
-    for col in df.columns:
-        if df[col].dtype == "object":
-            df[col] = df[col].str.slice(0, 2147483647)
+    # for col in df.columns:
+    #    if df[col].dtype == "object":
+    #        df[col] = df[col].str.slice(0, 2147483647)
 
     # Upload the data:
     print(f"Uploading {len(df)} rows to {table}...")
@@ -136,7 +141,10 @@ def prepare_dataclass(dataclass: dataclass) -> dict:
         "TRURISK_SCORE_FACTORS",
         "IP",
         "IPV6",
+        "QDS",
+        "QDS_FACTORS",
     ]
+
     DICT_FIELDS = [
         "CORRELATION",
         "CVSS",

qualyspy/sql/supported_uploads.py

@@ -63,7 +63,7 @@ def upload_vmdr_ags(ags: BaseList, cnxn: Connection) -> int:
     df = DataFrame([prepare_dataclass(ag) for ag in ags])
 
     # Upload the data:
-    return upload_data(df, "AssetGroups", cnxn, dtype=COLS)
+    return upload_data(df, "vmdr_assetgroups", cnxn, dtype=COLS)
 
 
 def upload_vmdr_kb(kbs: BaseList, cnxn: Connection) -> int:
@@ -115,16 +115,17 @@ def upload_vmdr_kb(kbs: BaseList, cnxn: Connection) -> int:
     df = DataFrame([prepare_dataclass(kb) for kb in kbs])
 
     # Upload the data:
-    return upload_data(df, "knowledgebase", cnxn, dtype=COLS)
+    return upload_data(df, "vmdr_knowledgebase", cnxn, dtype=COLS)
 
 
-def upload_vmdr_hosts(hosts: BaseList, cnxn: Connection) -> int:
+def upload_vmdr_hosts(hosts: BaseList, cnxn: Connection, is_hld: bool = False) -> int:
     """
     Upload data from vmdr.get_host_list() to SQL.
 
     Parameters:
         hosts (BaseList): The Host List to upload.
         cnxn (Connection): The Connection object to the SQL database.
+        is_hld (bool): If the data is from a Host List Detail pull. You can ignore this.
 
     Returns:
         int: The number of rows uploaded.
@@ -189,8 +190,10 @@ def upload_vmdr_hosts(hosts: BaseList, cnxn: Connection) -> int:
 
     df.drop(columns=["METADATA", "DNS_DATA", "DETECTION_LIST"], inplace=True)
 
-    # Upload the data:
-    return upload_data(df, "vmdr_hosts_list", cnxn, dtype=COLS)
+    # Upload the data, with table depdening on if it is a Host List Detail pull or not:
+    return upload_data(
+        df, "vmdr_hosts_list" if not is_hld else "vmdr_hld_hosts_list", cnxn, dtype=COLS
+    )
 
 
 def upload_vmdr_ips(ips: BaseList, cnxn: Connection) -> int:
@@ -206,17 +209,85 @@ def upload_vmdr_ips(ips: BaseList, cnxn: Connection) -> int:
     """
 
     COLS = {
-        "IP_OBJ": types.String(),
+        "IP": types.String(),
         "TYPE": types.String(),
     }
 
     # Convert the BaseList to a DataFrame:
-    df = DataFrame([prepare_dataclass(ip) for ip in ips], columns=["IP"])
+    df = DataFrame([str(ip) for ip in ips], columns=["IP"])
 
     # Add the TYPE column, which shows if it is a single IP or a range:
-    df["TYPE"] = df["IP"].apply(
-        lambda x: "Single IP" if "/" not in str(x) else "IP Range"
-    )
+    df["TYPE"] = df["IP"].apply(lambda x: "Single IP" if "/" not in x else "IP Range")
 
     # Upload the data:
     return upload_data(df, "vmdr_ips", cnxn, dtype=COLS)
+
+
+def upload_vmdr_hld(hld: BaseList, cnxn: Connection) -> int:
+    """
+    Upload data from vmdr.get_hld() to SQL.
+
+    Parameters:
+        hld (BaseList): The Host List to upload.
+        cnxn (Connection): The Connection object to the SQL database.
+
+    Returns:
+        int: The number of rows uploaded.
+
+    """
+
+    """
+    Get_hld and get_host_list technically return the same data. get_hld just
+    includes the DETECTION_LIST attribute. We can use the same upload function
+    for the host part, and then snip off the DETECTION_LIST attribute to upload
+    to a detections table.
+    """
+
+    # Isolate the detection lists. Since the Detection objects themselves
+    # have an ID attribute, we can use that to link them back to the host.
+    detections = BaseList()
+    for host in hld:
+        if host.DETECTION_LIST:
+            for detection in host.DETECTION_LIST:
+                detections.append(detection)
+
+    # upload_vmdr_hosts automatically ignores the DETECTION_LIST attribute,
+    # so we can use it here with the is_hld flag set to True to put the hosts
+    # in a different table than get_host_list.
+    hosts_uploaded = upload_vmdr_hosts(hld, cnxn, is_hld=True)
+    print(
+        f"Uploaded {hosts_uploaded} hosts to vmdr_hld_hosts_list. Moving to detections..."
+    )
+
+    COLS = {
+        "UNIQUE_VULN_ID": types.BigInteger(),
+        "QID": types.BigInteger(),
+        "TYPE": types.String(),
+        "SEVERITY": types.Integer(),
+        "STATUS": types.String(),
+        "SSL": types.Boolean(),
+        "RESULTS": types.String(),
+        "FIRST_FOUND_DATETIME": types.DateTime(),
+        "LAST_FOUND_DATETIME": types.DateTime(),
+        "QDS": types.Integer(),
+        "QDS_FACTORS": types.String(),
+        "TIMES_FOUND": types.Integer(),
+        "LAST_TEST_DATETIME": types.DateTime(),
+        "LAST_UPDATE_DATETIME": types.DateTime(),
+        "IS_IGNORED": types.Boolean(),
+        "IS_DISABLED": types.Boolean(),
+        "LAST_PROCESSED_DATETIME": types.DateTime(),
+        "LAST_FIXED_DATETIME": types.DateTime(),
+        "PORT": types.Integer(),
+        "PROTOCOL": types.String(),
+        "FQDN": types.String(),
+    }
+
+    # Convert the BaseList to a DataFrame:
+    df = DataFrame([prepare_dataclass(detection) for detection in detections])
+
+    # Set QDS to an integer:
+    df["QDS"] = df["QDS"].apply(lambda x: int(x) if x else None)
+
+    # Upload the data:
+    return upload_data(df, "vmdr_hld_detections", cnxn, dtype=COLS)

qualyspy/vmdr/data_classes/detection.py

@@ -11,6 +11,7 @@
 
 from .qds_factor import QDSFactor
 from .qds import QDS as qds
+from ..data_classes.lists import BaseList
 
 
 @dataclass(order=True)
@@ -117,6 +118,11 @@ class Detection:
         compare=False,
     )
 
+    ID: int = field(
+        metadata={"description": "The host ID of host the detection is on."},
+        default=None,
+    )
+
     def __post_init__(self):
         # convert the datetimes to datetime objects
         DATETIME_FIELDS = [
@@ -132,7 +138,7 @@ def __post_init__(self):
 
         BOOL_FIELDS = ["IS_IGNORED", "IS_DISABLED", "SSL"]
 
-        INT_FIELDS = ["UNIQUE_VULN_ID", "QID", "SEVERITY", "TIMES_FOUND", "PORT"]
+        INT_FIELDS = ["UNIQUE_VULN_ID", "QID", "SEVERITY", "TIMES_FOUND", "PORT", "ID"]
 
         for field in DATETIME_FIELDS:
             if (
@@ -173,20 +179,19 @@ def __post_init__(self):
 
         # convert the QDS factors to QDSFactor objects
         if self.QDS_FACTORS:
-            # if [QDS_FACTORS][QDS_FACTOR] is a list of dictionaries, itereate through each dictionary and convert it to a QDSFactor object
-            # if it is just one dictionary, convert it to a QDSFactor object
-            if isinstance(self.QDS_FACTORS["QDS_FACTOR"], list):
-                self.QDS_FACTORS = [
+            factors_bl = BaseList()
+            data = self.QDS_FACTORS["QDS_FACTOR"]
+
+            # Normalize QDS factors to a list for easier processing
+            if isinstance(data, dict):
+                data = [data]
+
+            for factor in data:
+                factors_bl.append(
                     QDSFactor(NAME=factor["@name"], VALUE=factor["#text"])
-                    for factor in self.QDS_FACTORS["QDS_FACTOR"]
-                ]
-            else:
-                self.QDS_FACTORS = [
-                    QDSFactor(
-                        NAME=self.QDS_FACTORS["QDS_FACTOR"]["@name"],
-                        VALUE=self.QDS_FACTORS["QDS_FACTOR"]["#text"],
-                    )
-                ]
+                )
+
+            self.QDS_FACTORS = factors_bl
 
     def __str__(self):
         # return str(self.UNIQUE_VULN_ID)

qualyspy/vmdr/data_classes/hosts.py

@@ -205,7 +205,7 @@ class VMDRHost:
         compare=False,
     )
 
-    DETECTION_LIST: Optional[Union[list[Detection], BaseList[Detection]]] = field(
+    DETECTION_LIST: Optional[BaseList[Detection]] = field(
         default=None,
         metadata={"description": "The detection list of the host."},
         compare=False,
@@ -328,7 +328,6 @@ def __post_init__(self):
             "ASSET_RISK_SCORE",
             "TRURISK_SCORE",
             "ASSET_CRITICALITY_SCORE",
-            "CLOUD_ACCOUNT_ID",
         ]
 
         if self.DNS_DATA:
@@ -450,17 +449,20 @@ def __post_init__(self):
 
         # check for a detections list and convert it to a BaseList of Detection objects (used in hld):
         if self.DETECTION_LIST:
-            if isinstance(self.DETECTION_LIST["DETECTION"], list):
-                self.DETECTION_LIST = BaseList(
-                    [
-                        Detection.from_dict(detection)
-                        for detection in self.DETECTION_LIST["DETECTION"]
-                    ]
-                )
-            else:
-                self.DETECTION_LIST = BaseList(
-                    [Detection.from_dict(self.DETECTION_LIST["DETECTION"])]
-                )
+
+            detections_bl = BaseList()
+            data = self.DETECTION_LIST["DETECTION"]
+
+            if isinstance(data, dict):
+                data = [data]
+
+            for detection in data:
+                # Append the host's ID attr to the detection dictionary
+                # to allow for a relationship:
+                detection["ID"] = self.ID
+                detections_bl.append(Detection.from_dict(detection))
+
+            self.DETECTION_LIST = detections_bl
 
     def __str__(self) -> str:
         """