diff --git a/update_versions.py b/update_versions.py
index 77d8b42ac5477ce58471ec21830ef57ca8f54870..593c1cdb0de55bb8e9446e761e82d2ffac4ebdcc 100644
--- a/update_versions.py
+++ b/update_versions.py
@@ -1,895 +1,920 @@
-# ============================================================
-# update_versions.py
-# ============================================================
-#
-# When run from the Autopsy build script, this script will:
-# - Clone Autopsy and checkout to the previous release tag
-# as found in the NEWS.txt file
-# - Auto-discover all modules and packages
-# - Run jdiff, comparing the current and previous modules
-# - Use jdiff's output to determine if each module
-# a) has no changes
-# b) has backwards compatible changes
-# c) has backwards incompatible changes
-# - Based off it's compatibility, updates each module's
-# a) Major version
-# b) Specification version
-# c) Implementation version
-# - Updates the dependencies on each module depending on the
-# updated version numbers
-#
-# Optionally, when run from the command line, one can provide the
-# desired tag to compare the current version to, the directory for
-# the current version of Autopsy, and whether to automatically
-# update the version numbers and dependencies.
-# ------------------------------------------------------------
-
-import errno
-import os
-import shutil
-import stat
-import subprocess
-import sys
-import traceback
-from os import remove, close
-from shutil import move
-from tempfile import mkstemp
-from xml.dom.minidom import parse, parseString
-
-# An Autopsy module object
-class Module:
- # Initialize it with a name, return code, and version numbers
- def __init__(self, name=None, ret=None, versions=None):
- self.name = name
- self.ret = ret
- self.versions = versions
- # As a string, the module should be it's name
- def __str__(self):
- return self.name
- def __repr__(self):
- return self.name
- # When compared to another module, the two are equal if the names are the same
- def __cmp__(self, other):
- if isinstance(other, Module):
- if self.name == other.name:
- return 0
- elif self.name < other.name:
- return -1
- else:
- return 1
- return 1
- def __eq__(self, other):
- if isinstance(other, Module):
- if self.name == other.name:
- return True
- return False
- def set_name(self, name):
- self.name = name
- def set_ret(self, ret):
- self.ret = ret
- def set_versions(self, versions):
- self.versions = versions
- def spec(self):
- return self.versions[0]
- def impl(self):
- return self.versions[1]
- def release(self):
- return self.versions[2]
-
-# Representation of the Specification version number
-class Spec:
- # Initialize specification number, where num is a string like x.y
- def __init__(self, num):
- l, r = num.split(".")
- self.left = int(l)
- self.right = int(r)
- def __str__(self):
- return self.get()
- def __cmp__(self, other):
- if isinstance(other, Spec):
- if self.left == other.left:
- if self.right == other.right:
- return 0
- if self.right < other.right:
- return -1
- return 1
- if self.left < other.left:
- return -1
- return 1
- elif isinstance(other, str):
- l, r = other.split(".")
- if self.left == int(l):
- if self.right == int(r):
- return 0
- if self.right < int(r):
- return -1
- return 1
- if self.left < int(l):
- return -1
- return 1
- return -1
-
- def overflow(self):
- return str(self.left + 1) + ".0"
- def increment(self):
- return str(self.left) + "." + str(self.right + 1)
- def get(self):
- return str(self.left) + "." + str(self.right)
- def set(self, num):
- if isinstance(num, str):
- l, r = num.split(".")
- self.left = int(l)
- self.right = int(r)
- elif isinstance(num, Spec):
- self.left = num.left
- self.right = num.right
- return self
-
-# ================================ #
-# Core Functions #
-# ================================ #
-
-# Given a list of modules and the names for each version, compare
-# the generated jdiff XML for each module and output the jdiff
-# JavaDocs.
-#
-# modules: the list of all modules both versions have in common
-# apiname_tag: the api name of the previous version, most likely the tag
-# apiname_cur: the api name of the current version, most likely "Current"
-#
-# returns the exit code from the modified jdiff.jar
-# return code 1 = error in jdiff
-# return code 100 = no changes
-# return code 101 = compatible changes
-# return code 102 = incompatible changes
-def compare_xml(module, apiname_tag, apiname_cur):
- global docdir
- make_dir(docdir)
- null_file = fix_path(os.path.abspath("./thirdparty/jdiff/v-custom/lib/Null.java"))
- jdiff = fix_path(os.path.abspath("./thirdparty/jdiff/v-custom/jdiff.jar"))
- oldapi = fix_path("build/jdiff-xml/" + apiname_tag + "-" + module.name)
- newapi = fix_path("build/jdiff-xml/" + apiname_cur + "-" + module.name)
- docs = fix_path(docdir + "/" + module.name)
- # Comments are strange. They look for a file with additional user comments in a
- # directory like docs/user_comments_for_xyz. The problem being that xyz is the
- # path to the new/old api. So xyz turns into multiple directories for us.
- # i.e. user_comments_for_build/jdiff-xml/[tag name]-[module name]_to_build/jdiff-xml
- comments = fix_path(docs + "/user_comments_for_build")
- jdiff_com = fix_path(comments + "/jdiff-xml")
- tag_comments = fix_path(jdiff_com + "/" + apiname_tag + "-" + module.name + "_to_build")
- jdiff_tag_com = fix_path(tag_comments + "/jdiff-xml")
- make_dir(docs)
- make_dir(comments)
- make_dir(jdiff_com)
- make_dir(tag_comments)
- make_dir(jdiff_tag_com)
- make_dir("jdiff-logs")
- log = open("jdiff-logs/COMPARE-" + module.name + ".log", "w")
- cmd = ["javadoc",
- "-doclet", "jdiff.JDiff",
- "-docletpath", jdiff,
- "-d", docs,
- "-oldapi", oldapi,
- "-newapi", newapi,
- "-script",
- null_file]
- jdiff = subprocess.Popen(cmd, stdout=log, stderr=log)
- jdiff.wait()
- log.close()
- code = jdiff.returncode
- print("Compared XML for " + module.name)
- if code == 100:
- print(" No API changes")
- elif code == 101:
- print(" API Changes are backwards compatible")
- elif code == 102:
- print(" API Changes are not backwards compatible")
- else:
- print(" *Error in XML, most likely an empty module")
- sys.stdout.flush()
- return code
-
-# Generate the jdiff xml for the given module
-# path: path to the autopsy source
-# module: Module object
-# name: api name for jdiff
-def gen_xml(path, modules, name):
- for module in modules:
- # If its the regression test, the source is in the "test" dir
- if module.name == "Testing":
- src = os.path.join(path, module.name, "test", "qa-functional", "src")
- else:
- src = os.path.join(path, module.name, "src")
- # xerces = os.path.abspath("./lib/xerces.jar")
- xml_out = fix_path(os.path.abspath("./build/jdiff-xml/" + name + "-" + module.name))
- jdiff = fix_path(os.path.abspath("./thirdparty/jdiff/v-custom/jdiff.jar"))
- make_dir("build/jdiff-xml")
- make_dir("jdiff-logs")
- log = open("jdiff-logs/GEN_XML-" + name + "-" + module.name + ".log", "w")
- cmd = ["javadoc",
- "-doclet", "jdiff.JDiff",
- "-docletpath", jdiff, # ;" + xerces, <-- previous problems required this
- "-apiname", xml_out, # leaving it in just in case it's needed once again
- "-sourcepath", fix_path(src)]
- cmd = cmd + get_packages(src)
- jdiff = subprocess.Popen(cmd, stdout=log, stderr=log)
- jdiff.wait()
- log.close()
- print("Generated XML for " + name + " " + module.name)
- sys.stdout.flush()
-
-# Find all the modules in the given path
-def find_modules(path):
- modules = []
- # Step into each folder in the given path and
- # see if it has manifest.mf - if so, it's a module
- for dir in os.listdir(path):
- directory = os.path.join(path, dir)
- if os.path.isdir(directory):
- for file in os.listdir(directory):
- if file == "manifest.mf":
- modules.append(Module(dir, None, None))
- return modules
-
-# Detects the differences between the source and tag modules
-def module_diff(source_modules, tag_modules):
- added_modules = [x for x in source_modules if x not in tag_modules]
- removed_modules = [x for x in tag_modules if x not in source_modules]
- similar_modules = [x for x in source_modules if x in tag_modules]
-
- added_modules = (added_modules if added_modules else [])
- removed_modules = (removed_modules if removed_modules else [])
- similar_modules = (similar_modules if similar_modules else [])
- return similar_modules, added_modules, removed_modules
-
-# Reads the previous tag from NEWS.txt
-def get_tag(sourcepath):
- news = open(sourcepath + "/NEWS.txt", "r")
- second_instance = False
- for line in news:
- if "----------------" in line:
- if second_instance:
- ver = line.split("VERSION ")[1]
- ver = ver.split(" -")[0]
- return "autopsy-" + ver
- else:
- second_instance = True
- continue
- news.close()
-
-
-# ========================================== #
-# Dependency Functions #
-# ========================================== #
-
-# Write a new XML file, copying all the lines from projectxml
-# and replacing the specification version for the code-name-base base
-# with the supplied specification version spec
-def set_dep_spec(projectxml, base, spec):
- print(" Updating Specification version..")
- orig = open(projectxml, "r")
- f, abs_path = mkstemp()
- new_file = open(abs_path, "w")
- found_base = False
- spacing = " "
- sopen = "<specification-version>"
- sclose = "</specification-version>\n"
- for line in orig:
- if base in line:
- found_base = True
- if found_base and sopen in line:
- update = spacing + sopen + str(spec) + sclose
- new_file.write(update)
- else:
- new_file.write(line)
- new_file.close()
- close(f)
- orig.close()
- remove(projectxml)
- move(abs_path, projectxml)
-
-# Write a new XML file, copying all the lines from projectxml
-# and replacing the release version for the code-name-base base
-# with the supplied release version
-def set_dep_release(projectxml, base, release):
- print(" Updating Release version..")
- orig = open(projectxml, "r")
- f, abs_path = mkstemp()
- new_file = open(abs_path, "w")
- found_base = False
- spacing = " "
- ropen = "<release-version>"
- rclose = "</release-version>\n"
- for line in orig:
- if base in line:
- found_base = True
- if found_base and ropen in line:
- update = spacing + ropen + str(release) + rclose
- new_file.write(update)
- else:
- new_file.write(line)
- new_file.close()
- close(f)
- orig.close()
- remove(projectxml)
- move(abs_path, projectxml)
-
-# Return the dependency versions in the XML dependency node
-def get_dep_versions(dep):
- run_dependency = dep.getElementsByTagName("run-dependency")[0]
- release_version = run_dependency.getElementsByTagName("release-version")
- if release_version:
- release_version = getTagText(release_version[0].childNodes)
- specification_version = run_dependency.getElementsByTagName("specification-version")
- if specification_version:
- specification_version = getTagText(specification_version[0].childNodes)
- return int(release_version), Spec(specification_version)
-
-# Given a code-name-base, see if it corresponds with any of our modules
-def get_module_from_base(modules, code_name_base):
- for module in modules:
- if "org.sleuthkit.autopsy." + module.name.lower() == code_name_base:
- return module
- return None # If it didn't match one of our modules
-
-# Check the text between two XML tags
-def getTagText(nodelist):
- for node in nodelist:
- if node.nodeType == node.TEXT_NODE:
- return node.data
-
-# Check the projectxml for a dependency on any module in modules
-def check_for_dependencies(projectxml, modules):
- dom = parse(projectxml)
- dep_list = dom.getElementsByTagName("dependency")
- for dep in dep_list:
- code_name_base = dep.getElementsByTagName("code-name-base")[0]
- code_name_base = getTagText(code_name_base.childNodes)
- module = get_module_from_base(modules, code_name_base)
- if module:
- print(" Found dependency on " + module.name)
- release, spec = get_dep_versions(dep)
- if release != module.release() and module.release() is not None:
- set_dep_release(projectxml, code_name_base, module.release())
- else: print(" Release version is correct")
- if spec != module.spec() and module.spec() is not None:
- set_dep_spec(projectxml, code_name_base, module.spec())
- else: print(" Specification version is correct")
-
-# Given the module and the source directory, return
-# the paths to the manifest and project properties files
-def get_dependency_file(module, source):
- projectxml = os.path.join(source, module.name, "nbproject", "project.xml")
- if os.path.isfile(projectxml):
- return projectxml
-
-# Verify/Update the dependencies for each module, basing the dependency
-# version number off the versions in each module
-def update_dependencies(modules, source):
- for module in modules:
- print("Checking the dependencies for " + module.name + "...")
- projectxml = get_dependency_file(module, source)
- if projectxml == None:
- print(" Error finding project xml file")
- else:
- other = [x for x in modules]
- check_for_dependencies(projectxml, other)
- sys.stdout.flush()
-
-# ======================================== #
-# Versioning Functions #
-# ======================================== #
-
-# Return the specification version in the given project.properties/manifest.mf file
-def get_specification(project, manifest):
- try:
- # Try to find it in the project file
- # it will be there if impl version is set to append automatically
- f = open(project, 'r')
- for line in f:
- if "spec.version.base" in line:
- return Spec(line.split("=")[1].strip())
- f.close()
- # If not found there, try the manifest file
- f = open(manifest, 'r')
- for line in f:
- if "OpenIDE-Module-Specification-Version:" in line:
- return Spec(line.split(": ")[1].strip())
- except:
- print("Error parsing Specification version for")
- print(project)
-
-# Set the specification version in the given project properties file
-# but if it can't be found there, set it in the manifest file
-def set_specification(project, manifest, num):
- try:
- # First try the project file
- f = open(project, 'r')
- for line in f:
- if "spec.version.base" in line:
- f.close()
- replace(project, line, "spec.version.base=" + str(num) + "\n")
- return
- f.close()
- # If it's not there, try the manifest file
- f = open(manifest, 'r')
- for line in f:
- if "OpenIDE-Module-Specification-Version:" in line:
- f.close()
- replace(manifest, line, "OpenIDE-Module-Specification-Version: " + str(num) + "\n")
- return
- # Otherwise we're out of luck
- print(" Error finding the Specification version to update")
- print(" " + manifest)
- f.close()
- except:
- print(" Error incrementing Specification version for")
- print(" " + project)
-
-# Return the implementation version in the given manifest.mf file
-def get_implementation(manifest):
- try:
- f = open(manifest, 'r')
- for line in f:
- if "OpenIDE-Module-Implementation-Version" in line:
- return int(line.split(": ")[1].strip())
- f.close()
- except:
- print("Error parsing Implementation version for")
- print(manifest)
-
-# Set the implementation version in the given manifest file
-def set_implementation(manifest, num):
- try:
- f = open(manifest, 'r')
- for line in f:
- if "OpenIDE-Module-Implementation-Version" in line:
- f.close()
- replace(manifest, line, "OpenIDE-Module-Implementation-Version: " + str(num) + "\n")
- return
- # If it isn't there, add it
- f.close()
- write_implementation(manifest, num)
- except:
- print(" Error incrementing Implementation version for")
- print(" " + manifest)
-
-# Rewrite the manifest file to include the implementation version
-def write_implementation(manifest, num):
- f = open(manifest, "r")
- contents = f.read()
- contents = contents[:-2] + "OpenIDE-Module-Implementation-Version: " + str(num) + "\n\n"
- f.close()
- f = open(manifest, "w")
- f.write(contents)
- f.close()
-
-# Return the release version in the given manifest.mf file
-def get_release(manifest):
- try:
- f = open(manifest, 'r')
- for line in f:
- if "OpenIDE-Module:" in line:
- return int(line.split("/")[1].strip())
- f.close()
- except:
- print("Error parsing Release version for")
- print(manifest)
-
-# Set the release version in the given manifest file
-def set_release(manifest, num):
- try:
- f = open(manifest, 'r')
- for line in f:
- if "OpenIDE-Module:" in line:
- f.close()
- index = line.index('/') - len(line) + 1
- newline = line[:index] + str(num)
- replace(manifest, line, newline + "\n")
- return
- print(" Error finding the release version to update")
- print(" " + manifest)
- f.close()
- except:
- print(" Error incrementing release version for")
- print(" " + manifest)
-
-# Given the module and the source directory, return
-# the paths to the manifest and project properties files
-def get_version_files(module, source):
- manifest = os.path.join(source, module.name, "manifest.mf")
- project = os.path.join(source, module.name, "nbproject", "project.properties")
- if os.path.isfile(manifest) and os.path.isfile(project):
- return manifest, project
-
-# Returns a the current version numbers for the module in source
-def get_versions(module, source):
- manifest, project = get_version_files(module, source)
- if manifest == None or project == None:
- print(" Error finding manifeset and project properties files")
- return
- spec = get_specification(project, manifest)
- impl = get_implementation(manifest)
- release = get_release(manifest)
- return [spec, impl, release]
-
-# Update the version numbers for every module in modules
-def update_versions(modules, source):
- for module in modules:
- versions = module.versions
- manifest, project = get_version_files(module, source)
- print("Updating " + module.name + "...")
- if manifest == None or project == None:
- print(" Error finding manifeset and project properties files")
- return
- if module.ret == 101:
- versions = [versions[0].set(versions[0].increment()), versions[1] + 1, versions[2]]
- set_specification(project, manifest, versions[0])
- set_implementation(manifest, versions[1])
- module.set_versions(versions)
- elif module.ret == 102:
- versions = [versions[0].set(versions[0].overflow()), versions[1] + 1, versions[2] + 1]
- set_specification(project, manifest, versions[0])
- set_implementation(manifest, versions[1])
- set_release(manifest, versions[2])
- module.set_versions(versions)
- elif module.ret == 100:
- versions = [versions[0], versions[1] + 1, versions[2]]
- set_implementation(manifest, versions[1])
- module.set_versions(versions)
- elif module.ret == None:
- versions = [Spec("1.0"), 1, 1]
- set_specification(project, manifest, versions[0])
- set_implementation(manifest, versions[1])
- set_release(manifest, versions[2])
- module.set_versions(versions)
- sys.stdout.flush()
-
-# Given a list of the added modules, remove the modules
-# which have the correct 'new module default' version number
-def remove_correct_added(modules):
- correct = [x for x in modules]
- for module in modules:
- if module.spec() == "1.0" or module.spec() == "0.0":
- if module.impl() == 1:
- if module.release() == 1 or module.release() == 0:
- correct.remove(module)
- return correct
-
-# ==================================== #
-# Helper Functions #
-# ==================================== #
-
-# Replace pattern with subst in given file
-def replace(file, pattern, subst):
- #Create temp file
- fh, abs_path = mkstemp()
- new_file = open(abs_path,'w')
- old_file = open(file)
- for line in old_file:
- new_file.write(line.replace(pattern, subst))
- #close temp file
- new_file.close()
- close(fh)
- old_file.close()
- #Remove original file
- remove(file)
- #Move new file
- move(abs_path, file)
-
-# Given a list of modules print the version numbers that need changing
-def print_version_updates(modules):
- f = open("gen_version.txt", "a")
- for module in modules:
- versions = module.versions
- if module.ret == 101:
- output = (module.name + ":\n")
- output += (" Current Specification version:\t" + str(versions[0]) + "\n")
- output += (" Updated Specification version:\t" + str(versions[0].increment()) + "\n")
- output += ("\n")
- output += (" Current Implementation version:\t" + str(versions[1]) + "\n")
- output += (" Updated Implementation version:\t" + str(versions[1] + 1) + "\n")
- output += ("\n")
- print(output)
- f.write(output)
- elif module.ret == 102:
- output = (module.name + ":\n")
- output += (" Current Specification version:\t" + str(versions[0]) + "\n")
- output += (" Updated Specification version:\t" + str(versions[0].overflow()) + "\n")
- output += ("\n")
- output += (" Current Implementation version:\t" + str(versions[1]) + "\n")
- output += (" Updated Implementation version:\t" + str(versions[1] + 1) + "\n")
- output += ("\n")
- output += (" Current Release version:\t\t" + str(versions[2]) + "\n")
- output += (" Updated Release version:\t\t" + str(versions[2] + 1) + "\n")
- output += ("\n")
- print(output)
- f.write(output)
- elif module.ret == 1:
- output = (module.name + ":\n")
- output += (" *Unable to detect necessary changes\n")
- output += (" Current Specification version:\t" + str(versions[0]) + "\n")
- output += (" Current Implementation version:\t" + str(versions[1]) + "\n")
- output += (" Current Release version:\t\t" + str(versions[2]) + "\n")
- output += ("\n")
- print(output)
- f.write(output)
- elif module.ret == 100:
- output = (module.name + ":\n")
- output += (" Current Implementation version:\t" + str(versions[1]) + "\n")
- output += (" Updated Implementation version:\t" + str(versions[1] + 1) + "\n")
- output += ("\n")
- print(output)
- f.write(output)
- elif module.ret is None:
- output = ("Added " + module.name + ":\n")
- if module.spec() != "1.0" and module.spec() != "0.0":
- output += (" Current Specification version:\t" + str(module.spec()) + "\n")
- output += (" Updated Specification version:\t1.0\n")
- output += ("\n")
- if module.impl() != 1:
- output += (" Current Implementation version:\t" + str(module.impl()) + "\n")
- output += (" Updated Implementation version:\t1\n")
- output += ("\n")
- if module.release() != 1 and module.release() != 0:
- output += (" Current Release version:\t\t" + str(module.release()) + "\n")
- output += (" Updated Release version:\t\t1\n")
- output += ("\n")
- print(output)
- f.write(output)
- sys.stdout.flush()
- f.close()
-
-# Changes cygwin paths to Windows
-def fix_path(path):
- if "cygdrive" in path:
- new_path = path[11:]
- return "C:/" + new_path
- else:
- return path
-
-# Print a 'title'
-def printt(title):
- print("\n" + title)
- lines = ""
- for letter in title:
- lines += "-"
- print(lines)
- sys.stdout.flush()
-
-# Get a list of package names in the given path
-# The path is expected to be of the form {base}/module/src
-#
-# NOTE: We currently only check for packages of the form
-# org.sleuthkit.autopsy.x
-# If we add other namespaces for commercial modules we will
-# have to add a check here
-def get_packages(path):
- packages = []
- package_path = os.path.join(path, "org", "sleuthkit", "autopsy")
- for folder in os.listdir(package_path):
- package_string = "org.sleuthkit.autopsy."
- packages.append(package_string + folder)
- return packages
-
-# Create the given directory, if it doesn't already exist
-def make_dir(dir):
- try:
- if not os.path.isdir(dir):
- os.mkdir(dir)
- if os.path.isdir(dir):
- return True
- return False
- except:
- print("Exception thrown when creating directory")
- return False
-
-# Delete the given directory, and make sure it is deleted
-def del_dir(dir):
- try:
- if os.path.isdir(dir):
- shutil.rmtree(dir, ignore_errors=False, onerror=handleRemoveReadonly)
- if os.path.isdir(dir):
- return False
- else:
- return True
- return True
- except:
- print("Exception thrown when deleting directory")
- traceback.print_exc()
- return False
-
-# Handle any permisson errors thrown by shutil.rmtree
-def handleRemoveReadonly(func, path, exc):
- excvalue = exc[1]
- if func in (os.rmdir, os.remove) and excvalue.errno == errno.EACCES:
- os.chmod(path, stat.S_IRWXU| stat.S_IRWXG| stat.S_IRWXO) # 0777
- func(path)
- else:
- raise
-
-# Run git clone and git checkout for the tag
-def do_git(tag, tag_dir):
- try:
- printt("Cloning Autopsy tag " + tag + " into dir " + tag_dir + " (this could take a while)...")
- subprocess.call(["git", "clone", "https://github.com/sleuthkit/autopsy.git", tag_dir],
- stdout=subprocess.PIPE)
- printt("Checking out tag " + tag + "...")
- subprocess.call(["git", "checkout", tag],
- stdout=subprocess.PIPE,
- cwd=tag_dir)
- return True
- except Exception as ex:
- print("Error cloning and checking out Autopsy: ", sys.exc_info()[0])
- print ex
- print("The terminal you are using most likely does not recognize git commands.")
- return False
-
-# Get the flags from argv
-def args():
- try:
- sys.argv.pop(0)
- while sys.argv:
- arg = sys.argv.pop(0)
- if arg == "-h" or arg == "--help":
- return 1
- elif arg == "-t" or arg == "--tag":
- global tag
- tag = sys.argv.pop(0)
- elif arg == "-s" or arg == "--source":
- global source
- source = sys.argv.pop(0)
- elif arg == "-d" or arg == "--dir":
- global docdir
- docdir = sys.argv.pop(0)
- elif arg == "-a" or arg == "--auto":
- global dry
- dry = False
- else:
- raise Exception()
- except:
- pass
-
-# Print script run info
-def printinfo():
- global tag
- global source
- global docdir
- global dry
- printt("Release script information:")
- if source is None:
- source = fix_path(os.path.abspath("."))
- print("Using source directory:\n " + source)
- if tag is None:
- tag = get_tag(source)
- print("Checking out to tag:\n " + tag)
- if docdir is None:
- docdir = fix_path(os.path.abspath("./jdiff-javadocs"))
- print("Generating jdiff JavaDocs in:\n " + docdir)
- if dry is True:
- print("Dry run: will not auto-update version numbers")
- sys.stdout.flush()
-
-# Print the script's usage/help
-def usage():
- return \
- """
- USAGE:
- Run this script to generate a jdiff XML summary for every module
- in the current Autopsy source and in a previous source specified
- by the given tag. Then, compare the XML files to see which modules
- need updated version numbers. If the dry run tag is not given, the
- module numbers will be automatically updated.
-
- OPTIONAL FLAGS:
- -t --tag The tag name in git. Otherwise the NEWS file in source
- will be used to determine the previous tag.
-
- -d --dir The output directory for the jdiff JavaDocs. If no
- directory is given, the default is /javadocs/{module}.
-
- -s --source The directory containing Autopsy's source code.
-
- -a --auto Automatically update version numbers (not dry).
-
- -h --help Prints this usage.
- """
-
-# ==================================== #
-# Main Functionality #
-# ==================================== #
-
-# Where the magic happens
-def main():
- global tag; global source; global docdir; global dry
- tag = None; source = None; docdir = None; dry = True
-
- ret = args()
- if ret:
- print(usage())
- return 0
- printinfo()
-
- # -----------------------------------------------
- # 1) Clone Autopsy, checkout to given tag/commit
- # 2) Get the modules in the clone and the source
- # 3) Generate the xml comparison
- # -----------------------------------------------
- if not del_dir("./build/" + tag):
- print("\n\n=========================================")
- print(" Failed to delete previous Autopsy clone.")
- print(" Unable to continue...")
- print("=========================================")
- return 1
- tag_dir = os.path.abspath("./build/" + tag)
- if not do_git(tag, tag_dir):
- return 1
- sys.stdout.flush()
-
- tag_modules = find_modules(tag_dir)
- source_modules = find_modules(source)
-
- printt("Generating jdiff XML reports...")
- apiname_tag = tag
- apiname_cur = "current"
- gen_xml(tag_dir, tag_modules, apiname_tag)
- gen_xml(source, source_modules, apiname_cur)
-
- printt("Deleting cloned Autopsy directory...")
- print("Clone successfully deleted" if del_dir(tag_dir) else "Failed to delete clone")
- sys.stdout.flush()
-
- # -----------------------------------------------------
- # 1) Seperate modules into added, similar, and removed
- # 2) Compare XML for each module
- # -----------------------------------------------------
- printt("Comparing modules found...")
- similar_modules, added_modules, removed_modules = module_diff(source_modules, tag_modules)
- if added_modules or removed_modules:
- for m in added_modules:
- print("+ Added " + m.name)
- sys.stdout.flush()
- for m in removed_modules:
- print("- Removed " + m.name)
- sys.stdout.flush()
- else:
- print("No added or removed modules")
- sys.stdout.flush()
-
- printt("Comparing jdiff outputs...")
- for module in similar_modules:
- module.set_ret(compare_xml(module, apiname_tag, apiname_cur))
-
- # ------------------------------------------------------------
- # 1) Do versioning
- # 2) Auto-update version numbers in files and the_modules list
- # 3) Auto-update dependencies
- # ------------------------------------------------------------
- printt("Auto-detecting version numbers and changes...")
- for module in added_modules:
- module.set_versions(get_versions(module, source))
- for module in similar_modules:
- module.set_versions(get_versions(module, source))
-
- added_modules = remove_correct_added(added_modules)
- the_modules = similar_modules + added_modules
- print_version_updates(the_modules)
-
- if not dry:
- printt("Auto-updating version numbers...")
- update_versions(the_modules, source)
- print("All auto-updates complete")
-
- printt("Detecting and auto-updating dependencies...")
- update_dependencies(the_modules, source)
-
- printt("Deleting jdiff XML...")
- xml_dir = os.path.abspath("./build/jdiff-xml")
- print("XML successfully deleted" if del_dir(xml_dir) else "Failed to delete XML")
-
- print("\n--- Script completed successfully ---")
- return 0
-
-# Start off the script
-if __name__ == "__main__":
- sys.exit(main())
\ No newline at end of file
+# ============================================================
+# update_versions.py
+# ============================================================
+#
+# When run from the Autopsy build script, this script will:
+# - Clone Autopsy and checkout to the previous release tag
+# as found in the NEWS.txt file
+# - Auto-discover all modules and packages
+# - Run jdiff, comparing the current and previous modules
+# - Use jdiff's output to determine if each module
+# a) has no changes
+# b) has backwards compatible changes
+# c) has backwards incompatible changes
+# - Based off it's compatibility, updates each module's
+# a) Major version
+# b) Specification version
+# c) Implementation version
+# - Updates the dependencies on each module depending on the
+# updated version numbers
+#
+# Optionally, when run from the command line, one can provide the
+# desired tag to compare the current version to, the directory for
+# the current version of Autopsy, and whether to automatically
+# update the version numbers and dependencies.
+# ------------------------------------------------------------
+
+import errno
+import os
+import shutil
+import stat
+import subprocess
+import sys
+import traceback
+from os import remove, close
+from shutil import move
+from tempfile import mkstemp
+from xml.dom.minidom import parse, parseString
+
+# An Autopsy module object
+class Module:
+ # Initialize it with a name, return code, and version numbers
+ def __init__(self, name=None, ret=None, versions=None):
+ self.name = name
+ self.ret = ret
+ self.versions = versions
+ # As a string, the module should be it's name
+ def __str__(self):
+ return self.name
+ def __repr__(self):
+ return self.name
+ # When compared to another module, the two are equal if the names are the same
+ def __cmp__(self, other):
+ if isinstance(other, Module):
+ if self.name == other.name:
+ return 0
+ elif self.name < other.name:
+ return -1
+ else:
+ return 1
+ return 1
+ def __eq__(self, other):
+ if isinstance(other, Module):
+ if self.name == other.name:
+ return True
+ return False
+ def set_name(self, name):
+ self.name = name
+ def set_ret(self, ret):
+ self.ret = ret
+ def set_versions(self, versions):
+ self.versions = versions
+ def spec(self):
+ return self.versions[0]
+ def impl(self):
+ return self.versions[1]
+ def release(self):
+ return self.versions[2]
+
+# Representation of the Specification version number
+class Spec:
+ # Initialize specification number, where num is a string like x.y
+ def __init__(self, num):
+ self.third = None
+ spec_nums = num.split(".")
+ if len(spec_nums) == 3:
+ final = spec_nums[2]
+ self.third = int(final)
+
+ l, r = spec_nums[0], spec_nums[1]
+
+ self.left = int(l)
+ self.right = int(r)
+
+ def __str__(self):
+ return self.get()
+ def __cmp__(self, other):
+ if isinstance(other, Spec):
+ if self.left == other.left:
+ if self.right == other.right:
+ return 0
+ if self.right < other.right:
+ return -1
+ return 1
+ if self.left < other.left:
+ return -1
+ return 1
+ elif isinstance(other, str):
+ l, r = other.split(".")
+ if self.left == int(l):
+ if self.right == int(r):
+ return 0
+ if self.right < int(r):
+ return -1
+ return 1
+ if self.left < int(l):
+ return -1
+ return 1
+ return -1
+
+ def overflow(self):
+ return str(self.left + 1) + ".0"
+ def increment(self):
+ return str(self.left) + "." + str(self.right + 1)
+ def get(self):
+ spec_str = str(self.left) + "." + str(self.right)
+ if self.third is not None:
+ spec_str += "." + str(self.final)
+ return spec_str
+ def set(self, num):
+ if isinstance(num, str):
+ l, r = num.split(".")
+ self.left = int(l)
+ self.right = int(r)
+ elif isinstance(num, Spec):
+ self.left = num.left
+ self.right = num.right
+ return self
+
+# ================================ #
+# Core Functions #
+# ================================ #
+
+# Given a list of modules and the names for each version, compare
+# the generated jdiff XML for each module and output the jdiff
+# JavaDocs.
+#
+# modules: the list of all modules both versions have in common
+# apiname_tag: the api name of the previous version, most likely the tag
+# apiname_cur: the api name of the current version, most likely "Current"
+#
+# returns the exit code from the modified jdiff.jar
+# return code 1 = error in jdiff
+# return code 100 = no changes
+# return code 101 = compatible changes
+# return code 102 = incompatible changes
+def compare_xml(module, apiname_tag, apiname_cur):
+ global docdir
+ make_dir(docdir)
+ null_file = fix_path(os.path.abspath("./thirdparty/jdiff/v-custom/lib/Null.java"))
+ jdiff = fix_path(os.path.abspath("./thirdparty/jdiff/v-custom/jdiff.jar"))
+ oldapi = fix_path("build/jdiff-xml/" + apiname_tag + "-" + module.name)
+ newapi = fix_path("build/jdiff-xml/" + apiname_cur + "-" + module.name)
+ docs = fix_path(docdir + "/" + module.name)
+ # Comments are strange. They look for a file with additional user comments in a
+ # directory like docs/user_comments_for_xyz. The problem being that xyz is the
+ # path to the new/old api. So xyz turns into multiple directories for us.
+ # i.e. user_comments_for_build/jdiff-xml/[tag name]-[module name]_to_build/jdiff-xml
+ comments = fix_path(docs + "/user_comments_for_build")
+ jdiff_com = fix_path(comments + "/jdiff-xml")
+ tag_comments = fix_path(jdiff_com + "/" + apiname_tag + "-" + module.name + "_to_build")
+ jdiff_tag_com = fix_path(tag_comments + "/jdiff-xml")
+
+ if not os.path.exists(jdiff):
+ print("JDIFF doesn't exist.")
+
+ make_dir(docs)
+ make_dir(comments)
+ make_dir(jdiff_com)
+ make_dir(tag_comments)
+ make_dir(jdiff_tag_com)
+ make_dir("jdiff-logs")
+ log = open("jdiff-logs/COMPARE-" + module.name + ".log", "w")
+ cmd = ["javadoc",
+ "-doclet", "jdiff.JDiff",
+ "-docletpath", jdiff,
+ "-d", docs,
+ "-oldapi", oldapi,
+ "-newapi", newapi,
+ "-script",
+ null_file]
+ jdiff = subprocess.Popen(cmd, stdout=log, stderr=log)
+ jdiff.wait()
+ log.close()
+ code = jdiff.returncode
+ print("Compared XML for " + module.name)
+ if code == 100:
+ print(" No API changes")
+ elif code == 101:
+ print(" API Changes are backwards compatible")
+ elif code == 102:
+ print(" API Changes are not backwards compatible")
+ else:
+ print(" *Error in XML, most likely an empty module")
+ sys.stdout.flush()
+ return code
+
+# Generate the jdiff xml for the given module
+# path: path to the autopsy source
+# module: Module object
+# name: api name for jdiff
+def gen_xml(path, modules, name):
+ for module in modules:
+ # If its the regression test, the source is in the "test" dir
+ if module.name == "Testing":
+ src = os.path.join(path, module.name, "test", "qa-functional", "src")
+ else:
+ src = os.path.join(path, module.name, "src")
+ # xerces = os.path.abspath("./lib/xerces.jar")
+ xml_out = fix_path(os.path.abspath("./build/jdiff-xml/" + name + "-" + module.name))
+ jdiff = fix_path(os.path.abspath("./thirdparty/jdiff/v-custom/jdiff.jar"))
+ make_dir("build/jdiff-xml")
+ make_dir("jdiff-logs")
+ log = open("jdiff-logs/GEN_XML-" + name + "-" + module.name + ".log", "w")
+ cmd = ["javadoc",
+ "-doclet", "jdiff.JDiff",
+ "-docletpath", jdiff, # ;" + xerces, <-- previous problems required this
+ "-apiname", xml_out, # leaving it in just in case it's needed once again
+ "-sourcepath", fix_path(src)]
+ cmd = cmd + get_packages(src)
+ jdiff = subprocess.Popen(cmd, stdout=log, stderr=log)
+ jdiff.wait()
+ log.close()
+ print("Generated XML for " + name + " " + module.name)
+ sys.stdout.flush()
+
+# Find all the modules in the given path
+def find_modules(path):
+ modules = []
+ # Step into each folder in the given path and
+ # see if it has manifest.mf - if so, it's a module
+ for dir in os.listdir(path):
+ directory = os.path.join(path, dir)
+ if os.path.isdir(directory):
+ for file in os.listdir(directory):
+ if file == "manifest.mf":
+ modules.append(Module(dir, None, None))
+ return modules
+
+# Detects the differences between the source and tag modules
+def module_diff(source_modules, tag_modules):
+ added_modules = [x for x in source_modules if x not in tag_modules]
+ removed_modules = [x for x in tag_modules if x not in source_modules]
+ similar_modules = [x for x in source_modules if x in tag_modules]
+
+ added_modules = (added_modules if added_modules else [])
+ removed_modules = (removed_modules if removed_modules else [])
+ similar_modules = (similar_modules if similar_modules else [])
+ return similar_modules, added_modules, removed_modules
+
+# Reads the previous tag from NEWS.txt
+def get_tag(sourcepath):
+ news = open(sourcepath + "/NEWS.txt", "r")
+ second_instance = False
+ for line in news:
+ if "----------------" in line:
+ if second_instance:
+ ver = line.split("VERSION ")[1]
+ ver = ver.split(" -")[0]
+ return ("autopsy-" + ver).strip()
+ else:
+ second_instance = True
+ continue
+ news.close()
+
+
+# ========================================== #
+# Dependency Functions #
+# ========================================== #
+
+# Write a new XML file, copying all the lines from projectxml
+# and replacing the specification version for the code-name-base base
+# with the supplied specification version spec
+def set_dep_spec(projectxml, base, spec):
+ print(" Updating Specification version..")
+ orig = open(projectxml, "r")
+ f, abs_path = mkstemp()
+ new_file = open(abs_path, "w")
+ found_base = False
+ spacing = " "
+ sopen = "<specification-version>"
+ sclose = "</specification-version>\n"
+ for line in orig:
+ if base in line:
+ found_base = True
+ if found_base and sopen in line:
+ update = spacing + sopen + str(spec) + sclose
+ new_file.write(update)
+ else:
+ new_file.write(line)
+ new_file.close()
+ close(f)
+ orig.close()
+ remove(projectxml)
+ move(abs_path, projectxml)
+
+# Write a new XML file, copying all the lines from projectxml
+# and replacing the release version for the code-name-base base
+# with the supplied release version
+def set_dep_release(projectxml, base, release):
+ print(" Updating Release version..")
+ orig = open(projectxml, "r")
+ f, abs_path = mkstemp()
+ new_file = open(abs_path, "w")
+ found_base = False
+ spacing = " "
+ ropen = "<release-version>"
+ rclose = "</release-version>\n"
+ for line in orig:
+ if base in line:
+ found_base = True
+ if found_base and ropen in line:
+ update = spacing + ropen + str(release) + rclose
+ new_file.write(update)
+ else:
+ new_file.write(line)
+ new_file.close()
+ close(f)
+ orig.close()
+ remove(projectxml)
+ move(abs_path, projectxml)
+
+# Return the dependency versions in the XML dependency node
+def get_dep_versions(dep):
+ run_dependency = dep.getElementsByTagName("run-dependency")[0]
+ release_version = run_dependency.getElementsByTagName("release-version")
+ if release_version:
+ release_version = getTagText(release_version[0].childNodes)
+ specification_version = run_dependency.getElementsByTagName("specification-version")
+ if specification_version:
+ specification_version = getTagText(specification_version[0].childNodes)
+ return int(release_version), Spec(specification_version)
+
+# Given a code-name-base, see if it corresponds with any of our modules
+def get_module_from_base(modules, code_name_base):
+ for module in modules:
+ if "org.sleuthkit.autopsy." + module.name.lower() == code_name_base:
+ return module
+ return None # If it didn't match one of our modules
+
+# Check the text between two XML tags
+def getTagText(nodelist):
+ for node in nodelist:
+ if node.nodeType == node.TEXT_NODE:
+ return node.data
+
+# Check the projectxml for a dependency on any module in modules
+def check_for_dependencies(projectxml, modules):
+ dom = parse(projectxml)
+ dep_list = dom.getElementsByTagName("dependency")
+ for dep in dep_list:
+ code_name_base = dep.getElementsByTagName("code-name-base")[0]
+ code_name_base = getTagText(code_name_base.childNodes)
+ module = get_module_from_base(modules, code_name_base)
+ if module:
+ print(" Found dependency on " + module.name)
+ release, spec = get_dep_versions(dep)
+ if release != module.release() and module.release() is not None:
+ set_dep_release(projectxml, code_name_base, module.release())
+ else: print(" Release version is correct")
+ if spec != module.spec() and module.spec() is not None:
+ set_dep_spec(projectxml, code_name_base, module.spec())
+ else: print(" Specification version is correct")
+
+# Given the module and the source directory, return
+# the paths to the manifest and project properties files
+def get_dependency_file(module, source):
+ projectxml = os.path.join(source, module.name, "nbproject", "project.xml")
+ if os.path.isfile(projectxml):
+ return projectxml
+
+# Verify/Update the dependencies for each module, basing the dependency
+# version number off the versions in each module
+def update_dependencies(modules, source):
+ for module in modules:
+ print("Checking the dependencies for " + module.name + "...")
+ projectxml = get_dependency_file(module, source)
+ if projectxml == None:
+ print(" Error finding project xml file")
+ else:
+ other = [x for x in modules]
+ check_for_dependencies(projectxml, other)
+ sys.stdout.flush()
+
+# ======================================== #
+# Versioning Functions #
+# ======================================== #
+
+# Return the specification version in the given project.properties/manifest.mf file
+def get_specification(project, manifest):
+ try:
+ # Try to find it in the project file
+ # it will be there if impl version is set to append automatically
+ f = open(project, 'r')
+ for line in f:
+ if "spec.version.base" in line:
+ return Spec(line.split("=")[1].strip())
+ f.close()
+ # If not found there, try the manifest file
+ f = open(manifest, 'r')
+ for line in f:
+ if "OpenIDE-Module-Specification-Version:" in line:
+ return Spec(line.split(": ")[1].strip())
+ except Exception as e:
+ print("Error parsing Specification version for")
+ print(project)
+ print(e)
+
+# Set the specification version in the given project properties file
+# but if it can't be found there, set it in the manifest file
+def set_specification(project, manifest, num):
+ try:
+ # First try the project file
+ f = open(project, 'r')
+ for line in f:
+ if "spec.version.base" in line:
+ f.close()
+ replace(project, line, "spec.version.base=" + str(num) + "\n")
+ return
+ f.close()
+ # If it's not there, try the manifest file
+ f = open(manifest, 'r')
+ for line in f:
+ if "OpenIDE-Module-Specification-Version:" in line:
+ f.close()
+ replace(manifest, line, "OpenIDE-Module-Specification-Version: " + str(num) + "\n")
+ return
+ # Otherwise we're out of luck
+ print(" Error finding the Specification version to update")
+ print(" " + manifest)
+ f.close()
+ except:
+ print(" Error incrementing Specification version for")
+ print(" " + project)
+
+# Return the implementation version in the given manifest.mf file
+def get_implementation(manifest):
+ try:
+ f = open(manifest, 'r')
+ for line in f:
+ if "OpenIDE-Module-Implementation-Version" in line:
+ return int(line.split(": ")[1].strip())
+ f.close()
+ except:
+ print("Error parsing Implementation version for")
+ print(manifest)
+
+# Set the implementation version in the given manifest file
+def set_implementation(manifest, num):
+ try:
+ f = open(manifest, 'r')
+ for line in f:
+ if "OpenIDE-Module-Implementation-Version" in line:
+ f.close()
+ replace(manifest, line, "OpenIDE-Module-Implementation-Version: " + str(num) + "\n")
+ return
+ # If it isn't there, add it
+ f.close()
+ write_implementation(manifest, num)
+ except:
+ print(" Error incrementing Implementation version for")
+ print(" " + manifest)
+
+# Rewrite the manifest file to include the implementation version
+def write_implementation(manifest, num):
+ f = open(manifest, "r")
+ contents = f.read()
+ contents = contents[:-2] + "OpenIDE-Module-Implementation-Version: " + str(num) + "\n\n"
+ f.close()
+ f = open(manifest, "w")
+ f.write(contents)
+ f.close()
+
+# Return the release version in the given manifest.mf file
+def get_release(manifest):
+ try:
+ f = open(manifest, 'r')
+ for line in f:
+ if "OpenIDE-Module:" in line:
+ return int(line.split("/")[1].strip())
+ f.close()
+ except:
+ #print("Error parsing Release version for")
+ #print(manifest)
+ return 0
+
+# Set the release version in the given manifest file
+def set_release(manifest, num):
+ try:
+ f = open(manifest, 'r')
+ for line in f:
+ if "OpenIDE-Module:" in line:
+ f.close()
+ index = line.index('/') - len(line) + 1
+ newline = line[:index] + str(num)
+ replace(manifest, line, newline + "\n")
+ return
+ print(" Error finding the release version to update")
+ print(" " + manifest)
+ f.close()
+ except:
+ print(" Error incrementing release version for")
+ print(" " + manifest)
+
+# Given the module and the source directory, return
+# the paths to the manifest and project properties files
+def get_version_files(module, source):
+ manifest = os.path.join(source, module.name, "manifest.mf")
+ project = os.path.join(source, module.name, "nbproject", "project.properties")
+ if os.path.isfile(manifest) and os.path.isfile(project):
+ return manifest, project
+
+# Returns a the current version numbers for the module in source
+def get_versions(module, source):
+ manifest, project = get_version_files(module, source)
+ if manifest == None or project == None:
+ print(" Error finding manifeset and project properties files")
+ return
+ spec = get_specification(project, manifest)
+ impl = get_implementation(manifest)
+ release = get_release(manifest)
+ return [spec, impl, release]
+
+# Update the version numbers for every module in modules
+def update_versions(modules, source):
+ for module in modules:
+ versions = module.versions
+ manifest, project = get_version_files(module, source)
+ print("Updating " + module.name + "...")
+ if manifest == None or project == None:
+ print(" Error finding manifeset and project properties files")
+ return
+ if module.ret == 101:
+ versions = [versions[0].set(versions[0].increment()), versions[1] + 1, versions[2]]
+ set_specification(project, manifest, versions[0])
+ set_implementation(manifest, versions[1])
+ module.set_versions(versions)
+ elif module.ret == 102:
+ versions = [versions[0].set(versions[0].overflow()), versions[1] + 1, versions[2] + 1]
+ set_specification(project, manifest, versions[0])
+ set_implementation(manifest, versions[1])
+ set_release(manifest, versions[2])
+ module.set_versions(versions)
+ elif module.ret == 100:
+ versions = [versions[0], versions[1] + 1, versions[2]]
+ set_implementation(manifest, versions[1])
+ module.set_versions(versions)
+ elif module.ret == None:
+ versions = [Spec("1.0"), 1, 1]
+ set_specification(project, manifest, versions[0])
+ set_implementation(manifest, versions[1])
+ set_release(manifest, versions[2])
+ module.set_versions(versions)
+ sys.stdout.flush()
+
+# Given a list of the added modules, remove the modules
+# which have the correct 'new module default' version number
+def remove_correct_added(modules):
+ correct = [x for x in modules]
+ for module in modules:
+ if module.spec() == "1.0" or module.spec() == "0.0":
+ if module.impl() == 1:
+ if module.release() == 1 or module.release() == 0:
+ correct.remove(module)
+ return correct
+
+# ==================================== #
+# Helper Functions #
+# ==================================== #
+
+# Replace pattern with subst in given file
+def replace(file, pattern, subst):
+ #Create temp file
+ fh, abs_path = mkstemp()
+ new_file = open(abs_path,'w')
+ old_file = open(file)
+ for line in old_file:
+ new_file.write(line.replace(pattern, subst))
+ #close temp file
+ new_file.close()
+ close(fh)
+ old_file.close()
+ #Remove original file
+ remove(file)
+ #Move new file
+ move(abs_path, file)
+
+# Given a list of modules print the version numbers that need changing
+def print_version_updates(modules):
+ f = open("gen_version.txt", "a")
+ for module in modules:
+ versions = module.versions
+ if module.ret == 101:
+ output = (module.name + ":\n")
+ output += (" Current Specification version:\t" + str(versions[0]) + "\n")
+ output += (" Updated Specification version:\t" + str(versions[0].increment()) + "\n")
+ output += ("\n")
+ output += (" Current Implementation version:\t" + str(versions[1]) + "\n")
+ output += (" Updated Implementation version:\t" + str(versions[1] + 1) + "\n")
+ output += ("\n")
+ print(output)
+ sys.stdout.flush()
+ f.write(output)
+ elif module.ret == 102:
+ output = (module.name + ":\n")
+ output += (" Current Specification version:\t" + str(versions[0]) + "\n")
+ output += (" Updated Specification version:\t" + str(versions[0].overflow()) + "\n")
+ output += ("\n")
+ output += (" Current Implementation version:\t" + str(versions[1]) + "\n")
+ output += (" Updated Implementation version:\t" + str(versions[1] + 1) + "\n")
+ output += ("\n")
+ output += (" Current Release version:\t\t" + str(versions[2]) + "\n")
+ output += (" Updated Release version:\t\t" + str(versions[2] + 1) + "\n")
+ output += ("\n")
+ print(output)
+ sys.stdout.flush()
+ f.write(output)
+ elif module.ret == 1:
+ output = (module.name + ":\n")
+ output += (" *Unable to detect necessary changes\n")
+ output += (" Current Specification version:\t" + str(versions[0]) + "\n")
+ output += (" Current Implementation version:\t" + str(versions[1]) + "\n")
+ output += (" Current Release version:\t\t" + str(versions[2]) + "\n")
+ output += ("\n")
+ print(output)
+ f.write(output)
+ sys.stdout.flush()
+ elif module.ret == 100:
+ output = (module.name + ":\n")
+ if versions[1] is None:
+ output += (" No Implementation version.\n")
+ else:
+ output += (" Current Implementation version:\t" + str(versions[1]) + "\n")
+ output += (" Updated Implementation version:\t" + str(versions[1] + 1) + "\n")
+ output += ("\n")
+ print(output)
+ sys.stdout.flush()
+ f.write(output)
+ elif module.ret is None:
+ output = ("Added " + module.name + ":\n")
+ if module.spec() != "1.0" and module.spec() != "0.0":
+ output += (" Current Specification version:\t" + str(module.spec()) + "\n")
+ output += (" Updated Specification version:\t1.0\n")
+ output += ("\n")
+ if module.impl() != 1:
+ output += (" Current Implementation version:\t" + str(module.impl()) + "\n")
+ output += (" Updated Implementation version:\t1\n")
+ output += ("\n")
+ if module.release() != 1 and module.release() != 0:
+ output += (" Current Release version:\t\t" + str(module.release()) + "\n")
+ output += (" Updated Release version:\t\t1\n")
+ output += ("\n")
+ print(output)
+ sys.stdout.flush()
+ f.write(output)
+ sys.stdout.flush()
+ f.close()
+
+# Changes cygwin paths to Windows
+def fix_path(path):
+ if "cygdrive" in path:
+ new_path = path[11:]
+ return "C:/" + new_path
+ else:
+ return path
+
+# Print a 'title'
+def printt(title):
+ print("\n" + title)
+ lines = ""
+ for letter in title:
+ lines += "-"
+ print(lines)
+ sys.stdout.flush()
+
+# Get a list of package names in the given path
+# The path is expected to be of the form {base}/module/src
+#
+# NOTE: We currently only check for packages of the form
+# org.sleuthkit.autopsy.x
+# If we add other namespaces for commercial modules we will
+# have to add a check here
+def get_packages(path):
+ packages = []
+ package_path = os.path.join(path, "org", "sleuthkit", "autopsy")
+ for folder in os.listdir(package_path):
+ package_string = "org.sleuthkit.autopsy."
+ packages.append(package_string + folder)
+ return packages
+
+# Create the given directory, if it doesn't already exist
+def make_dir(dir):
+ try:
+ if not os.path.isdir(dir):
+ os.mkdir(dir)
+ if os.path.isdir(dir):
+ return True
+ return False
+ except:
+ print("Exception thrown when creating directory")
+ return False
+
+# Delete the given directory, and make sure it is deleted
+def del_dir(dir):
+ try:
+ if os.path.isdir(dir):
+ shutil.rmtree(dir, ignore_errors=False, onerror=handleRemoveReadonly)
+ if os.path.isdir(dir):
+ return False
+ else:
+ return True
+ return True
+ except:
+ print("Exception thrown when deleting directory")
+ traceback.print_exc()
+ return False
+
+# Handle any permisson errors thrown by shutil.rmtree
+def handleRemoveReadonly(func, path, exc):
+ excvalue = exc[1]
+ if func in (os.rmdir, os.remove) and excvalue.errno == errno.EACCES:
+ os.chmod(path, stat.S_IRWXU| stat.S_IRWXG| stat.S_IRWXO) # 0777
+ func(path)
+ else:
+ raise
+
+# Run git clone and git checkout for the tag
+def do_git(tag, tag_dir):
+ try:
+ printt("Cloning Autopsy tag " + tag + " into dir " + tag_dir + " (this could take a while)...")
+ subprocess.call(["git", "clone", "https://github.com/sleuthkit/autopsy.git", tag_dir],
+ stdout=subprocess.PIPE)
+ printt("Checking out tag " + tag + "...")
+ subprocess.call(["git", "checkout", tag],
+ stdout=subprocess.PIPE,
+ cwd=tag_dir)
+ return True
+ except Exception as ex:
+ print("Error cloning and checking out Autopsy: ", sys.exc_info()[0])
+ print(str(ex))
+ print("The terminal you are using most likely does not recognize git commands.")
+ return False
+
+# Get the flags from argv
+def args():
+ try:
+ sys.argv.pop(0)
+ while sys.argv:
+ arg = sys.argv.pop(0)
+ if arg == "-h" or arg == "--help":
+ return 1
+ elif arg == "-t" or arg == "--tag":
+ global tag
+ tag = sys.argv.pop(0)
+ elif arg == "-s" or arg == "--source":
+ global source
+ source = sys.argv.pop(0)
+ elif arg == "-d" or arg == "--dir":
+ global docdir
+ docdir = sys.argv.pop(0)
+ elif arg == "-a" or arg == "--auto":
+ global dry
+ dry = False
+ else:
+ raise Exception()
+ except:
+ pass
+
+# Print script run info
+def printinfo():
+ global tag
+ global source
+ global docdir
+ global dry
+ printt("Release script information:")
+ if source is None:
+ source = fix_path(os.path.abspath("."))
+ print("Using source directory:\n " + source)
+ if tag is None:
+ tag = get_tag(source)
+ print("Checking out to tag:\n " + tag)
+ if docdir is None:
+ docdir = fix_path(os.path.abspath("./jdiff-javadocs"))
+ print("Generating jdiff JavaDocs in:\n " + docdir)
+ if dry is True:
+ print("Dry run: will not auto-update version numbers")
+ sys.stdout.flush()
+
+# Print the script's usage/help
+def usage():
+ return \
+ """
+ USAGE:
+ Run this script to generate a jdiff XML summary for every module
+ in the current Autopsy source and in a previous source specified
+ by the given tag. Then, compare the XML files to see which modules
+ need updated version numbers. If the dry run tag is not given, the
+ module numbers will be automatically updated.
+
+ OPTIONAL FLAGS:
+ -t --tag The tag name in git. Otherwise the NEWS file in source
+ will be used to determine the previous tag.
+
+ -d --dir The output directory for the jdiff JavaDocs. If no
+ directory is given, the default is /javadocs/{module}.
+
+ -s --source The directory containing Autopsy's source code.
+
+ -a --auto Automatically update version numbers (not dry).
+
+ -h --help Prints this usage.
+ """
+
+# ==================================== #
+# Main Functionality #
+# ==================================== #
+
+# Where the magic happens
+def main():
+ global tag; global source; global docdir; global dry
+ tag = None; source = None; docdir = None; dry = True
+
+ ret = args()
+ if ret:
+ print(usage())
+ return 0
+ printinfo()
+
+ # -----------------------------------------------
+ # 1) Clone Autopsy, checkout to given tag/commit
+ # 2) Get the modules in the clone and the source
+ # 3) Generate the xml comparison
+ # -----------------------------------------------
+ if not del_dir("./build/" + tag):
+ print("\n\n=========================================")
+ print(" Failed to delete previous Autopsy clone.")
+ print(" Unable to continue...")
+ print("=========================================")
+ return 1
+ tag_dir = os.path.abspath("./build/" + tag)
+ if not do_git(tag, tag_dir):
+ return 1
+ sys.stdout.flush()
+
+ tag_modules = find_modules(tag_dir)
+ source_modules = find_modules(source)
+
+ printt("Generating jdiff XML reports...")
+ apiname_tag = tag
+ apiname_cur = "current"
+ gen_xml(tag_dir, tag_modules, apiname_tag)
+ gen_xml(source, source_modules, apiname_cur)
+
+ printt("Deleting cloned Autopsy directory...")
+ print("Clone successfully deleted" if del_dir(tag_dir) else "Failed to delete clone")
+ sys.stdout.flush()
+
+ # -----------------------------------------------------
+ # 1) Seperate modules into added, similar, and removed
+ # 2) Compare XML for each module
+ # -----------------------------------------------------
+ printt("Comparing modules found...")
+ similar_modules, added_modules, removed_modules = module_diff(source_modules, tag_modules)
+ if added_modules or removed_modules:
+ for m in added_modules:
+ print("+ Added " + m.name)
+ sys.stdout.flush()
+ for m in removed_modules:
+ print("- Removed " + m.name)
+ sys.stdout.flush()
+ else:
+ print("No added or removed modules")
+ sys.stdout.flush()
+
+ printt("Comparing jdiff outputs...")
+ for module in similar_modules:
+ module.set_ret(compare_xml(module, apiname_tag, apiname_cur))
+
+ # ------------------------------------------------------------
+ # 1) Do versioning
+ # 2) Auto-update version numbers in files and the_modules list
+ # 3) Auto-update dependencies
+ # ------------------------------------------------------------
+ printt("Auto-detecting version numbers and changes...")
+ for module in added_modules:
+ module.set_versions(get_versions(module, source))
+ for module in similar_modules:
+ module.set_versions(get_versions(module, source))
+
+ added_modules = remove_correct_added(added_modules)
+ the_modules = similar_modules + added_modules
+ print_version_updates(the_modules)
+
+ if not dry:
+ printt("Auto-updating version numbers...")
+ update_versions(the_modules, source)
+ print("All auto-updates complete")
+
+ printt("Detecting and auto-updating dependencies...")
+ update_dependencies(the_modules, source)
+
+ printt("Deleting jdiff XML...")
+ xml_dir = os.path.abspath("./build/jdiff-xml")
+ print("XML successfully deleted" if del_dir(xml_dir) else "Failed to delete XML")
+
+ print("\n--- Script completed successfully ---")
+ return 0
+
+# Start off the script
+if __name__ == "__main__":
+ sys.exit(main())