Merge pull request #990 from oh2fih/master

bin/cve_refs.py

@@ -19,7 +19,6 @@ sys.path.append(os.path.join(runPath, ".."))
 from lib.Config import Configuration
 
 try:
-
     r = Configuration.getRedisRefConnection()
 except:
     sys.exit(1)

bin/search.py

@@ -504,7 +504,6 @@ if vSearch:
 
 # Search text in summary
 if summary_text:
-
     for cveid in getCVEIDs(limit=nlimit):
         item = cves.getcve(cveid=cveid)
         if "cvss" in item:
@@ -521,7 +520,6 @@ if summary_text:
                 else:
                     print(json.dumps(item, sort_keys=True, default=json_util.default))
             else:
-
                 date_n_days_ago = datetime.now() - timedelta(days=last_ndays)
                 # print(item['Published'])
                 # print(type (item['Published']))

lib/ApiRequests.py

@@ -79,7 +79,6 @@ class JSONApiRequest(ApiRequest):
             )
             return reason, 400
         else:
-
             results = database_connection.query_docs(**self.body)
 
             self.logger.debug(

lib/CVEs.py

@@ -38,7 +38,6 @@ class CveHandler(object):
         via4lookup=False,
         subscorelookup=False,
     ):
-
         self.collectionname = collection
         self.rankinglookup = rankinglookup
         self.namelookup = namelookup
@@ -104,7 +103,6 @@ class CveHandler(object):
         return e
 
     def getranking(self, cpeid=None, loosy=True):
-
         if cpeid is None:
             return False
 

lib/DatabaseLayer.py

@@ -38,6 +38,7 @@ mongo_version = db.command("buildinfo")["versionArray"]
 # to be removed in future with the conditional statements
 # and use allow_disk_use by default
 
+
 # Functions
 def sanitize(x):
     if type(x) == pymongo.cursor.Cursor:

lib/DatabasePlugins/config.py

@@ -9,7 +9,6 @@ class DatabasePluginNotFound(Exception):
 
 class DatabasePluginLoader(object):
     def __init__(self):
-
         mods = [
             {plugin: _local_vars[plugin]}
             for plugin in _local_vars

lib/DatabasePlugins/mongodb.py

@@ -337,7 +337,6 @@ class MongoPlugin(DatabasePluginBase):
         search_val = request_values["search[value]"]
 
         if search_val != "":
-
             try:
                 regex = re.compile(search_val, re.IGNORECASE)
 

lib/DatabaseSchemaChecker.py

@@ -21,7 +21,6 @@ class SchemaChecker(object):
         self.logger = logging.getLogger("SchemaChecker")
 
     def validate_schema(self):
-
         if hasattr(self.dbh.connection, "store_schema"):
             try:
                 if (
@@ -56,7 +55,6 @@ class SchemaChecker(object):
 
     def update(self):
         try:
-
             current_record = list(self.dbh.connection.store_schema.find({}))
 
             if len(current_record) != 0:

lib/DownloadHandler.py

@@ -167,7 +167,6 @@ class DownloadHandler(ABC):
         x = 0
 
         while not_Done:
-
             current_q_len = self.queue.qsize()
 
             if x % 10 == 0:
@@ -179,7 +178,6 @@ class DownloadHandler(ABC):
                 )
 
             if current_q_len != 0:
-
                 if current_q_len != q_len:
                     q_len = current_q_len
                     dif = max_len - q_len
@@ -292,7 +290,11 @@ class DownloadHandler(ABC):
             with open(filename, "wb") as f:
                 f.write(gzip.GzipFile(fileobj=buf).read())
 
-        elif content_type == "application/json" or content_type == "application/xml":
+        elif (
+            content_type == "application/json"
+            or content_type == "application/xml"
+            or content_type == "text/xml"
+        ):
             filename = os.path.join(wd, url.split("/")[-1])
             self.logger.debug("Saving file to: {}".format(filename))
 

lib/JSONFileHandler.py

@@ -20,7 +20,6 @@ class JSONFileHandler(DownloadHandler):
         return "<< JSONFileHandler:{} >>".format(self.feed_type)
 
     def file_to_queue(self, file_tuple):
-
         working_dir, filename = file_tuple
 
         # adjust the interval counter for debug logging when updating

lib/LogHandler.py

@@ -70,7 +70,6 @@ class HelperLogger(logging.Logger):
     }
 
     def __init__(self, name, level=logging.NOTSET):
-
         super().__init__(name, level)
 
     def debug(self, msg, *args, **kwargs):

lib/Sources_process.py

@@ -655,7 +655,6 @@ class VIADownloads(JSONFileHandler):
         self.logger = logging.getLogger("VIADownloads")
 
     def file_to_queue(self, file_tuple):
-
         working_dir, filename = file_tuple
 
         for cve in self.ijson_handler.fetch(filename=filename, prefix=self.prefix):
@@ -685,7 +684,6 @@ class VIADownloads(JSONFileHandler):
             )
 
     def process_item(self, item):
-
         if self.is_update:
             self.queue.put(
                 DatabaseAction(
@@ -737,7 +735,6 @@ class CAPECDownloads(XMLFileHandler):
         self.parser.setContentHandler(self.ch)
 
     def file_to_queue(self, file_tuple):
-
         working_dir, filename = file_tuple
 
         self.parser.parse(filename)
@@ -790,7 +787,6 @@ class CWEDownloads(XMLFileHandler):
         self.parser.setContentHandler(self.ch)
 
     def file_to_queue(self, file_tuple):
-
         working_dir, filename = file_tuple
 
         self.parser.parse(filename)
@@ -853,7 +849,6 @@ class CPERedisBrowser(object):
 
         start_time = time.time()
         for e in tqdm(self.cpes, desc="Inserting CPE's in redis"):
-
             value = e["cpe_2_2"]
             (prefix, cpeversion, cpetype, vendor, product, version) = value.split(
                 ":", 5
@@ -883,7 +878,6 @@ MongoAddIndex = namedtuple("MongoAddIndex", "index name")
 
 class DatabaseIndexer(object):
     def __init__(self):
-
         self.indexes = {
             "cpe": [
                 MongoUniqueIndex(index=[("id", ASCENDING)], name="id", unique=True),
@@ -934,7 +928,6 @@ class DatabaseIndexer(object):
         self.logger = logging.getLogger("DatabaseIndexer")
 
     def create_indexes(self, collection=None):
-
         if collection is not None:
             try:
                 for each in self.indexes[collection]:

lib/Toolkit.py

@@ -107,7 +107,6 @@ def generate_title(cpe):
     cpe_split = cpe.split(":")
     # Do a very basic test to see if the CPE is valid
     if len(cpe_split) == 13:
-
         # Combine vendor, product and version
         title = " ".join(cpe_split[3:6])
 

lib/XMLFileHandler.py

@@ -14,7 +14,6 @@ class XMLFileHandler(DownloadHandler):
         return "<< XMLFileHandler:{} >>".format(self.feed_type)
 
     def process_item(self, item):
-
         if self.is_update:
             self.queue.put(
                 DatabaseAction(

lib/content_handlers.py

@@ -69,7 +69,6 @@ class CapecHandler(ContentHandler):
         self.execution_flow = defaultdict(dict)
 
     def startElement(self, name, attrs):
-
         if name == "Attack_Pattern_Catalog":
             self.Attack_Pattern_Catalog_tag = True
         if name == "Attack_Patterns" and self.Attack_Pattern_Catalog_tag:
@@ -222,7 +221,6 @@ class CapecHandler(ContentHandler):
                     )
 
             elif self.taxonomy_name == "WASC":
-
                 if "/" in self.entry_name_ch:
                     url = "http://projects.webappsec.org/{}".format(
                         self.entry_name_ch.replace("/", " and ").replace(" ", "-")
@@ -379,7 +377,6 @@ class CWEHandler(ContentHandler):
         self.alternate_terms = False
 
     def startElement(self, name, attrs):
-
         if name == "Weakness":
             self.weakness_tag = True
             self.statement = ""

lib/cpe_conversion.py

@@ -10,6 +10,7 @@
 
 import re
 
+
 # Convert cpe2.2 url encoded to cpe2.3 char escaped
 # cpe:2.3:o:cisco:ios:12.2%281%29 to cpe:2.3:o:cisco:ios:12.2\(1\)
 def unquote(cpe):

lib/cpelist.py

@@ -36,7 +36,6 @@ class CPEList:
     # insert to database
     def insert(self, cpe, cpeType, comments=None):
         try:
-
             if comments is not None:
                 comments = comments
             else:

lib/db_action.py

@@ -4,11 +4,9 @@ from pymongo import InsertOne, UpdateOne
 
 
 class DatabaseAction(object):
-
     actions = collections.namedtuple("Actions", "InsertOne UpdateOne")(0, 1)
 
     def __init__(self, action, collection, doc):
-
         self.action = action
         self.collection = collection
         self.doc = doc

lib/redis_q.py

@@ -33,7 +33,6 @@ class RedisQueue(object):
         return self._key
 
     def get_full_list(self):
-
         entries = self.__db.lrange(self.key, 0, -1)
 
         self.__db.delete(self.key)

sbin/db_blacklist.py

@@ -54,6 +54,7 @@ args = argparser.parse_args()
 # Variables
 collection = "blacklist"
 
+
 # Functions
 def importBlacklist(importFile):
     oList = CPEList(collection, args)