Merge pull request #222 from 0x41424142/refactor

Added PM-patchcatalog and sql uploads, small refactorings

docs/authentication.md

@@ -20,6 +20,8 @@ with BasicAuth(<username>,<password>, platform='qg1') as auth:
 >>>qualysdk.exceptions.Exceptions.AuthTypeMismatchError: Auth type mismatch. Expected token but got basic.
 ```
 
+>**Head's Up!**: Automatic rate limit respecting, described below, is not available for Patch Management API calls. Qualys does not return the headers necessary to determine rate limit windows for these endpoints.
+
 Both authentication objects also support automatic rate limit respecting. The SDK will warn you as you get close to an API endpoint's limit and automatically sleep until the limit is lifted, continuing the call afterwards:
 
 ```plaintext

docs/patch.md

@@ -29,6 +29,8 @@ You can use any of the endpoints currently supported:
 | ```get_asset_count``` | Returns the number of assets for a given platform that match ```query``` and ```havingQuery```. |
 | ```lookup_host_uuids``` | Returns a list of tuples, containing host UUIDs for a given list of asset IDs. |
 | ```get_patch_catalog``` | Returns the patch catalog for a given platform according to ```patchId```. |
+| ```get_packages_in_linux_patch``` | Returns the packages associated with a Linux patch. |
+| ```get_products_in_windows_patch``` | Returns the products associated with a Windows patch. |
 
 ## Get PM Version API
 
@@ -686,6 +688,90 @@ catalog = get_patch_catalog(
 ]
 ```
 
+## Get Packages Associated with Linux Patches API
+
+```get_packages_in_linux_patch``` returns the packages associated with a Linux patch.
+
+If a ```BaseList``` or a ```list``` of patch IDs is passed, the function will use threading to speed up the process.
+
+> <span style="color: red; font-weight: bold;">Warning:</span> You should filter down the patches as much as possible before passing them into this function. If you bulk-pass in a lot of patches, you will almost certainly hit a rate limit. **PM APIs do not return the headers necessary for the SDK to auto-recover from rate limits.**
+
+
+|Parameter| Possible Values |Description| Required|
+|--|--|--|--|
+|```auth```|```qualysdk.auth.TokenAuth``` | Authentication object | ✅ |
+| ```patchId``` | ```Union[str, BaseList[str]]``` | The ID(s) of the patch to get packages for | ✅ |
+| ```threads``` | ```int=5``` | The number of threads to use for the lookup. | ❌ |
+| ```filter``` | ```str``` | The QQL filter to search for packages | ❌ |
+| ```pageNumber``` | ```int``` | The page number to return. The SDK will handle pagination for you. Users can use this if ```page_count``` is 1 to pull a specific page. | ❌ |
+| ```pageSize``` | ```int=10``` | The number of packages to return per page | ❌ |
+
+```py
+from qualysdk.auth import TokenAuth
+from qualysdk.pm import get_packages_in_linux_patch, get_patches
+
+auth = TokenAuth(<username>, <password>, platform='qg1')
+
+# Get some Linux patches:
+patches = get_patches(
+  auth, 
+  platform='linux', 
+  attributes="id",
+  query="vendorSeverity:Critical"
+)
+
+# Get the packages for the patches:
+packages = get_packages_in_linux_patch(
+  auth, 
+  [patch.id for patch in patches]
+)
+>>>PackageDetail(
+  packageName='minidlna_1.3.0+dfsg-2+deb11u2', 
+  architecture='noarch', 
+  patchId='48e7d965-5f86-3118-a35f-b8fd1463e6b0'
+)
+```
+
+## Get Products Associated with Windows Patches API
+
+```get_products_in_windows_patch``` returns the products associated with a Windows patch.
+
+If a ```BaseList``` or a ```list``` of patch IDs is passed, the function will use threading to speed up the process.
+
+> <span style="color: red; font-weight: bold;">Warning:</span> You should filter down the patches as much as possible before passing them into this function. If you bulk-pass in a lot of patches, you will almost certainly hit a rate limit. **PM APIs do not return the headers necessary for the SDK to auto-recover from rate limits.**
+
+
+|Parameter| Possible Values |Description| Required|
+|--|--|--|--|
+|```auth```|```qualysdk.auth.TokenAuth``` | Authentication object | ✅ |
+| ```patchId``` | ```Union[str, BaseList[str]]``` | The ID(s) of the patch to get products for | ✅ |
+| ```threads``` | ```int=5``` | The number of threads to use for the lookup. | ❌ |
+
+```py
+from qualysdk.auth import TokenAuth
+from qualysdk.pm import get_products_in_windows_patch, get_patches
+
+auth = TokenAuth(<username>, <password>, platform='qg1')
+
+# Get some Windows patches:
+patches = get_patches(
+  auth, 
+  platform='windows', 
+  attributes="id",
+  query="vendorSeverity:Critical"
+)
+
+# Get the products for the patches:
+products = get_products_in_windows_patch(
+  auth, 
+  [patch.id for patch in patches]
+)
+>>>AssociatedProduct(
+  product=['Adobe Audition 2024 24 x64'], 
+  patchId='2c1649c0-a18a-3f77-8c52-a6ea297ab295'
+)
+```
+
 
 ## ```qualysdk-pm``` CLI tool
 

docs/sql.md

@@ -140,6 +140,8 @@ The final optional parameter is ```table_name```. If you want to specify a custo
 | ```upload_pm_assets``` | Patch Management | ```pm.get_assets()``` | ```pm_assets``` |
 | ```upload_pm_assetids_to_uuids``` | Patch Management | ```pm.lookup_host_uuids()``` | ```pm_assetids_to_uuids``` |
 | ```upload_pm_patch_catalog``` | Patch Management | ```pm.get_patch_catalog()``` | ```pm_patch_catalog``` |
+| ```upload_pm_linux_packages``` | Patch Management | ```pm.get_packages_in_linux_patch()``` | ```pm_linux_packages``` |
+| ```upload_pm_windows_products``` | Patch Management | ```pm.get_products_in_windows_patch()``` | ```pm_windows_products``` |
 | ```upload_cert_certs``` | Certificate View | ```cert.list_certs()``` | ```cert_certs``` for certificates and ```cert_assets``` for assets (key = certs.id -> assets.certId) |
 
 

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "qualysdk"
-version = "0.2.4"
+version = "0.2.5"
 description = "SDK for interacting with Qualys APIs, across most modules the platform offers."
 authors = ["0x41424142 <jake@jakelindsay.uk>", "0x4A616B65 <jake.lindsay@thermofisher.com>"]
 maintainers = ["Jake Lindsay <jake@jakelindsay.uk>"]

qualysdk/auth/basic.py

@@ -95,7 +95,7 @@ def test_login(self, return_ratelimit: bool = False) -> Union[dict, None]:
         if self.platform != "qg1":
             url = f"https://qualysapi.{self.platform}.apps.qualys.com/msp/about.php"
         else:
-            url = f"https://qualysapi.qualys.com/msp/about.php"
+            url = "https://qualysapi.qualys.com/msp/about.php"
 
         """Requires basic auth. JWT is not supported for this endpoint."""
         r = get(url, auth=(self.username, self.password))

qualysdk/base/call_api.py

@@ -272,7 +272,8 @@ def call_api(
             and int(response.headers["X-RateLimit-Remaining"]) < 10
         ) or response.status_code == 429:
             # Rate limit reached:
-            if int(response.headers["X-RateLimit-Remaining"]) == 0:
+            # Qualys does not return X-RateLimit headers for PM as of 12-2024. Sigh...
+            if module != "pm" and int(response.headers["X-RateLimit-Remaining"]) == 0:
                 # Call API again for the X-RateLimit-ToWait-Sec header.
                 # Qualys sometimes only includes this header when the rate limit is reached and retried:
                 response = request(
@@ -299,18 +300,18 @@ def call_api(
                     to_wait = 3601  # Default to 1h 1s if no header is present.
 
                 print(
-                    "WARNING: You have reached the rate limit for this endpoint. "
-                    + f"qualysdk will automatically sleep for {to_wait} seconds and try again at approximately {datetime.now() + timedelta(seconds=to_wait)}."
+                    f"WARNING: You have reached the rate limit for this endpoint. qualysdk will automatically sleep for {to_wait} seconds and try again at approximately {datetime.now() + timedelta(seconds=to_wait)}."
                 )
                 sleep(to_wait)
                 # Go to next iteration of the loop to try again:
                 continue
-
+            # Qualys does not return X-RateLimit headers for PM. Sigh...
+            elif module == "pm":
+                return response
             else:
                 # Almost at rate limit:
                 print(
-                    f"Warning: This endpoint will accept {response.headers['X-RateLimit-Remaining']} more calls before rate limiting you."
-                    + f" qualysdk will automatically sleep once remaining calls hits 0."
+                    f"Warning: This endpoint will accept {response.headers['X-RateLimit-Remaining']} more calls before rate limiting you. qualysdk will automatically sleep once remaining calls hits 0."
                 )
 
         return response

qualysdk/base/call_schemas/pm_schema.py

@@ -180,6 +180,35 @@
                 "pagination": False,
                 "auth_type": "token",
             },
+            "get_packages_in_linux_patch": {
+                "endpoint": "/pm/v1/patchcatalog/patch/packages",
+                "method": ["GET"],
+                "valid_params": [
+                    "patchUuid",
+                    "patchId",
+                    "filter",
+                    "pageNumber",
+                    "pageSize",
+                ],
+                "valid_POST_data": [],
+                "use_requests_json_data": False,
+                "return_type": "json",
+                "pagination": True,
+                "auth_type": "token",
+            },
+            "get_products_in_windows_patch": {
+                "endpoint": "/pm/v1/patchcatalog/patch/{placeholder}/products",
+                "method": ["GET"],
+                "valid_params": [
+                    "placeholder",
+                    "patchId",
+                ],
+                "valid_POST_data": [],
+                "use_requests_json_data": False,
+                "return_type": "json",
+                "pagination": False,
+                "auth_type": "token",
+            },
         },
     }
 )

qualysdk/base/printutils.py

@@ -5,7 +5,13 @@
 disable_stdout -> bool: Disable printing.
 """
 
-import sys, os
+import sys
+import os
 
-disable_stdout = lambda: setattr(sys, "stdout", open(os.devnull, "w"))
-enable_stdout = lambda: setattr(sys, "stdout", sys.__stdout__)
+
+def disable_stdout():
+    setattr(sys, "stdout", open(os.devnull, "w"))
+
+
+def enable_stdout():
+    setattr(sys, "stdout", sys.__stdout__)

qualysdk/cloud_agent/data_classes/Agent.py

@@ -115,21 +115,23 @@ def __post_init__(self):
         ]
         IP_ADDRESS_FIELDS = ["agentInfo_connectedFrom"]
 
-        for field in DATE_FIELDS:
-            if getattr(self, field):
-                setattr(self, field, datetime.fromisoformat(getattr(self, field)))
+        for date_field in DATE_FIELDS:
+            if getattr(self, date_field):
+                setattr(
+                    self, date_field, datetime.fromisoformat(getattr(self, date_field))
+                )
 
-        for field in INT_FIELDS:
-            if getattr(self, field):
-                setattr(self, field, int(getattr(self, field)))
+        for int_field in INT_FIELDS:
+            if getattr(self, int_field):
+                setattr(self, int_field, int(getattr(self, int_field)))
 
-        for field in FLOAT_FIELDS:
-            if getattr(self, field):
-                setattr(self, field, float(getattr(self, field)))
+        for float_field in FLOAT_FIELDS:
+            if getattr(self, float_field):
+                setattr(self, float_field, float(getattr(self, float_field)))
 
-        for field in IP_ADDRESS_FIELDS:
-            if getattr(self, field):
-                setattr(self, field, ip_address(getattr(self, field)))
+        for ip_field in IP_ADDRESS_FIELDS:
+            if getattr(self, ip_field):
+                setattr(self, field, ip_address(getattr(self, ip_field)))
 
         # Before setting none fields, let's build the BaseList objects
         if self.tags:
@@ -265,9 +267,9 @@ def __post_init__(self):
             else:
                 setattr(self, "networkInterface", None)
 
-        for field in TO_NONE_FIELDS:
-            if getattr(self, field) == {}:
-                setattr(self, field, None)
+        for none_field in TO_NONE_FIELDS:
+            if getattr(self, none_field) == {}:
+                setattr(self, none_field, None)
 
         if "isDockerHost" in asdict(self).keys():
             setattr(self, "isDockerHost", self.isDockerHost == "true")

qualysdk/gav/hosts.py

@@ -158,6 +158,7 @@ class Host(BaseClass):
     cloudProvider_region: Optional[str] = None
     cloudProvider_spotInstance: Optional[bool] = None
     cloudProvider_subnetId: Optional[str] = None
+    cloudProvider_tags: Optional[Union[str, List[str], BaseList[str]]] = None
     cloudProvider_vpcId: Optional[str] = None
     # Azure:
     cloudProvider_imageOffer: Optional[str] = None
@@ -523,6 +524,7 @@ def __post_init__(self):
                                 if tag.get("key")
                                 else f"{tag.get('name')}:{tag.get('value')}"
                             )
+                            bl.append(s)
                         setattr(self, "cloudProvider_tags", bl)
 
                 else:

qualysdk/pm/__init__.py

@@ -16,4 +16,8 @@
 from .version import get_version
 from .patches import get_patches, get_patch_count
 from .assets import get_assets, lookup_host_uuids
-from .patchcatalog import get_patch_catalog
+from .patchcatalog import (
+    get_patch_catalog,
+    get_packages_in_linux_patch,
+    get_products_in_windows_patch,
+)

qualysdk/pm/assets.py

@@ -5,7 +5,9 @@
 
 from typing import Union, Literal, overload
 
-from .base.threading_backend import _get_patches_or_assets as get_assets_backend
+from .base.assets_patches_threading_backend import (
+    _get_patches_or_assets as get_assets_backend,
+)
 from .data_classes.PMAsset import Asset
 from ..auth.token import TokenAuth
 from ..base.base_list import BaseList

qualysdk/pm/base/threading_backend.py → qualysdk/pm/base/assets_patches_threading_backend.py

@@ -7,7 +7,7 @@
 
 from ..data_classes.Patch import Patch
 from ..data_classes.PMAsset import Asset
-from ..base.page_limit import check_page_size_limit
+from .page_limit import check_page_size_limit
 from ...auth.token import TokenAuth
 from ...base.call_api import call_api
 from ...base.base_list import BaseList

qualysdk/pm/data_classes/AssociatedProduct.py

@@ -0,0 +1,31 @@
+"""
+Contains AssociatedProduct data class for products under a Qualys
+Windows patch.
+"""
+
+from dataclasses import dataclass
+
+from ...base.base_class import BaseClass
+from ...base.base_list import BaseList
+
+
+@dataclass
+class AssociatedProduct(BaseClass):
+    """
+    A data class representing a product associated with a Windows patch.
+
+    product: BaseList[str]
+        The product name(s) associated with the patch.
+    patchId: str
+        The UUID of the associated patch.
+    """
+
+    product: BaseList[str] = None
+    patchId: str = None
+
+    def __str__(self):
+        return f"{str(self.product)}"
+
+    def __post_init__(self):
+        if self.product:
+            setattr(self, "product", BaseList(self.product))

qualysdk/pm/data_classes/CatalogPatch.py

@@ -31,6 +31,7 @@ class PackageDetail(BaseClass):
 
     packageName: str = None
     architecture: str = None
+    patchId: str = None
 
     def __str__(self):
         return f"{self.packageName} ({self.architecture})"

qualysdk/pm/data_classes/__init__.py

@@ -8,3 +8,4 @@
 from .PMRun import PMRun
 from .Patch import Patch
 from .CatalogPatch import CatalogPatch
+from .AssociatedProduct import AssociatedProduct

qualysdk/pm/jobs.py

@@ -198,7 +198,7 @@ def list_jobs(
                 ),
             ]
 
-            print(f"Spawned threads for both Windows and Linux jobs...")
+            print("Spawned threads for both Windows and Linux jobs...")
 
             for thread in threads:
                 thread.start()