-
Brian Carrier authoredBrian Carrier authored
Code owners
Assign users and groups as approvers for specific file changes. Learn more.
updateAndBuildAll.py 9.01 KiB
# Copyright (c) 2017 Basis Technology.
#
# This software is distributed under the Common Public License 1.0
#
# Updates the TSK dependency repos (libewf, etc.), compiles them, and
# compiles various TSK platforms using the current branch
import codecs
import datetime
import logging
import os
import re
import shutil
import subprocess
import sys
import getopt
from sys import platform as _platform
import time
import traceback
MSBUILD_PATH = os.path.normpath("c:/Program Files (x86)/MSBuild/14.0/Bin/MSBuild.exe")
CURRENT_PATH = os.getcwd()
# save the build log in the output directory
LOG_PATH = os.path.join(CURRENT_PATH, 'output', time.strftime("%Y.%m.%d-%H.%M.%S"))
MINIMAL = False
def pullAndBuildAllDependencies(depBranch):
'''
Compile libewf, libvhdi, libvmdk.
Args:
depBranch: String, which branch to compile (currently only support master)
'''
# Passed is a global variable that gets set to false
# When an error occurs
global passed
passed = True
# get the LIBEWF_HOME, LIBVHDI_HOME, LIBVMDH_HOME
ewfHome = os.getenv("LIBEWF_HOME", "C:\\libewf_64bit")
vhdiHome = os.getenv("LIBVHDI_HOME", "C:\\libvhdi_64bit")
vmdkHome = os.getenv("LIBVMDK_HOME", "C:\\libvmdk_64bit\\libvmdk")
# check if ewfHome, vhdiHome or vmdhHome exits
checkPathExist(ewfHome)
checkPathExist(vhdiHome)
checkPathExist(vmdkHome)
# git update libewf, libvhdi and libvmdk
if(passed):
gitPull(ewfHome, "libewf_64bit", depBranch)
if(passed):
gitPull(vhdiHome, "libvhdi_64bit", depBranch)
if(passed):
gitPull(vmdkHome, "libvmdk_64bit", depBranch)
if not MINIMAL:
# build 32-bit of libewf, libvhdi, libvmdk and TSK
if(passed):
buildDependentLibs(ewfHome, 32, "libewf")
if(passed):
buildDependentLibs(vhdiHome, 32, "libvhdi")
if(passed):
buildDependentLibs(vmdkHome, 32, "libvmdk")
# build 64-bit of libewf, libvhdi, libvmdk and TSK
if(passed):
buildDependentLibs(ewfHome, 64, "libewf")
if(passed):
buildDependentLibs(vhdiHome, 64, "libvhdi")
if(passed):
buildDependentLibs(vmdkHome, 64, "libvmdk")
def buildTSKAll():
if not MINIMAL:
if(passed):
buildTSK(32, "Release")
if(passed):
buildTSK(32, "Release_NoLibs")
if(passed):
buildTSK(32, "Release_PostgreSQL")
if(passed):
buildTSK(64, "Release")
if(passed):
buildTSK(64, "Release_NoLibs")
if(passed):
buildTSK(64, "Release_PostgreSQL")
def checkPathExist(path):
global passed
if not os.path.exists(path):
print (path + " not exist.")
sys.stdout.flush()
passed = False
def gitPull(libHome, repo, branch):
'''
Pull the latest code.
Args:
libHome: according the environment variable to get the location
repo String of repository ("libewf_64bit", "libvmdk_64bit" or "libvhdi_64bit" which one to pull
branch: String, which branch to pull
'''
global SYS
global passed
gppth = os.path.join(LOG_PATH, "GitPullOutput" + repo + ".txt")
gpout = open(gppth, 'a')
print("Resetting " + repo)
sys.stdout.flush()
call = ["git", "reset", "--hard"]
ret = subprocess.call(call, stdout=sys.stdout, cwd=libHome)
if ret != 0:
passed = False
return
print("Checking out " + branch)
sys.stdout.flush()
call = ["git", "checkout", branch]
ret = subprocess.call(call, stdout=sys.stdout, cwd=libHome)
if ret != 0:
passed = False
return
call = ["git", "pull"]
print("Pulling " + repo + "/" + branch)
sys.stdout.flush()
ret = subprocess.call(call, stdout=sys.stdout, cwd=libHome)
if ret != 0:
passed = False
gpout.close()
if passed:
print("Update " + repo + " successfully.")
else:
print("Update " + repo + " failed.")
def buildDependentLibs(libHome, wPlatform, targetDll):
'''
build libewf.dll, libvhdi.dll and libvmdk.dll
'''
global passed
passed = True
print("Building " + str(wPlatform) + "-bit " + targetDll)
sys.stdout.flush()
target = "Release"
if wPlatform == 64:
dllFile = os.path.join(libHome, "msvscpp", "x64", target, targetDll +".dll")
elif wPlatform == 32:
dllFile = os.path.join(libHome,"msvscpp",target,targetDll + ".dll")
else:
print("Invalid platform")
sys.stdout.flush()
passed = False
return
if (os.path.isfile(dllFile)):
os.remove(dllFile)
os.chdir(os.path.join(libHome,"msvscpp"))
vs = []
vs.append(MSBUILD_PATH)
vs.append(os.path.join(targetDll + ".sln"))
vs.append("/p:configuration=" + target)
if wPlatform == 64:
vs.append("/p:platform=x64")
elif wPlatform == 32:
vs.append("/p:platform=Win32")
vs.append("/v:quiet")
vs.append("/t:build")
outputFile = os.path.join(LOG_PATH, targetDll + "Output.txt")
VSout = open(outputFile, 'w')
ret = subprocess.call(vs, stdout=sys.stdout)
errorCode = ret
VSout.close()
if ret > 0:
failed_proj = os.system("grep 'Done Building Project' " + outputFile + " | grep vcxproj |grep FAILED |wc -l |cut -f1 -d' '")
failed_pyewf = os.system("grep 'Done Building Project' " + outputFile + " | grep pyewf |grep FAILED |grep pywc -l |cut -f1 -d' '")
if failed_proj == failed_pyewf:
errorCode = 0
if errorCode != 0 or not os.path.exists(dllFile) or os.path.getctime(dllFile) < (time.time() - 2 * 60): # the new dll should not be 2 mins old
print(targetDll + " " + str(wPlatform) + "-bit C++ failed to build.\n")
print("return code: " + str(ret) + "\tdll file: " + dllFile + "\tcreated time: " + str(os.path.getctime(dllFile)))
sys.stdout.flush()
passed = False
os.chdir(CURRENT_PATH)
return
else:
print("Build " + str(wPlatform) + "-bit " + targetDll + " successfully")
os.chdir(CURRENT_PATH)
def buildTSK(wPlatform, target):
'''
Build C++ sleuthkit library
'''
global passed
print ("Building TSK " + str(wPlatform) + "-bit " + target + " build.")
sys.stdout.flush()
TSK_HOME = os.getenv("TSK_HOME",False)
if not TSK_HOME:
print("Please set the TSK_HOME environment variable")
sys.exit(1)
else:
os.chdir(os.path.join(os.getenv("TSK_HOME"),"win32"))
vs = []
vs.append(MSBUILD_PATH)
vs.append(os.path.join("tsk-win.sln"))
vs.append("/p:configuration=" + target)
if wPlatform == 64:
vs.append("/p:platform=x64")
elif wPlatform == 32:
vs.append("/p:platform=Win32")
else:
print("Invalid platform")
sys.stdout.flush()
passed = False
return
vs.append("/v:quiet")
vs.append("/t:clean")
vs.append("/t:build")
outputFile = os.path.join(LOG_PATH, "TSKOutput.txt")
VSout = open(outputFile, 'w')
ret = subprocess.call(vs, stdout=sys.stdout)
VSout.close()
if ret != 0:
print("ret = " + str(ret))
print(vs)
print("LIBTSK " + str(wPlatform) + "-bit C++ failed to build.\n")
sys.stdout.flush()
passed = False
return
def usage():
'''
Print out how to use this script.
'''
print('Usage: python3 updateAndBuildLibs.py [[-h | --help, -b <branch> | --branch=<branch>, -m | --minimal]')
print('branch: Branch for dependencies (master is default)')
print('-m,--minimal: Build 64-bit PostgreSQL only')
sys.stdout.flush()
sys.exit(1)
def main():
depBranch = 'master'
global MINIMAL
try:
opts, args = getopt.getopt(sys.argv[1:],"mhb:",['help','minimal','branch='])
except getopt.GetoptError as err:
print(err)
usage()
sys.exit(2)
for o,a in opts:
if o in ("-m","--minimal"):
MINIMAL = True
elif o in ("-b","--branch"):
depBranch = a
elif o in ("-h","--help"):
usage()
system.exit(2)
if not os.path.exists(LOG_PATH):
os.makedirs(LOG_PATH)
if not os.path.exists(MSBUILD_PATH):
print("MS_BUILD Does not exist")
sys.stdout.flush()
pullAndBuildAllDependencies(depBranch)
buildTSKAll()
class OS:
LINUX, MAC, WIN, CYGWIN = range(4)
if __name__ == "__main__":
global SYS
if _platform == "linux" or _platform == "linux2":
SYS = OS.LINUX
elif _platform == "darwin":
SYS = OS.MAC
elif _platform == "win32":
SYS = OS.WIN
elif _platform == "cygwin":
SYS = OS.CYGWIN
global passed
if SYS is OS.WIN or SYS is OS.CYGWIN:
passed = True
main()
else:
passed = False
print("We only support Windows and Cygwin at this time.")
sys.stdout.flush()
if (passed):
sys.exit(0)
else:
sys.exit(1)
#/cygdrive/c/Program\ Files\ \(x86\)/MSBuild/14.0/Bin/MSBuild.exe libewf.sln /p:Configuration=Release /p:platform=x64 /t:clean /t:libewf_dll /m /clp:ErrorsOnly /nologo