Skip to content
Snippets Groups Projects
Unverified Commit 09167a1b authored by Richard Cordovano's avatar Richard Cordovano Committed by GitHub
Browse files

Merge pull request #5819 from gdicristofaro/6113-gpxFileIngest 

6113 gpx file ingest
parents 0ce76727 84688113
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
InternalPythonModules/GPX_Module/GPX_Parser_Module.py
+ 152
− 130
View file @ 09167a1b
...@@ -44,7 +44,6 @@ ...@@ -44,7 +44,6 @@
from org.sleuthkit.autopsy.datamodel import ContentUtils from org.sleuthkit.autopsy.datamodel import ContentUtils
from org.sleuthkit.autopsy.ingest import IngestModule from org.sleuthkit.autopsy.ingest import IngestModule
from org.sleuthkit.autopsy.ingest.IngestModule import IngestModuleException from org.sleuthkit.autopsy.ingest.IngestModule import IngestModuleException
from org.sleuthkit.autopsy.ingest import DataSourceIngestModule
from org.sleuthkit.autopsy.ingest import FileIngestModule from org.sleuthkit.autopsy.ingest import FileIngestModule
from org.sleuthkit.autopsy.ingest import IngestModuleFactoryAdapter from org.sleuthkit.autopsy.ingest import IngestModuleFactoryAdapter
from org.sleuthkit.autopsy.ingest import IngestMessage from org.sleuthkit.autopsy.ingest import IngestMessage
...@@ -60,12 +59,17 @@ ...@@ -60,12 +59,17 @@
import gpxpy.gpx import gpxpy.gpx
import gpxpy.parser import gpxpy.parser
# to get a random filename to prevent race conditions
import uuid
# Factory that defines the name and details of the module and allows Autopsy # Factory that defines the name and details of the module and allows Autopsy
# to create instances of the modules that will do the analysis. # to create instances of the modules that will do the analysis.
class GPXParserDataSourceIngestModuleFactory(IngestModuleFactoryAdapter):
class GPXParserFileIngestModuleFactory(IngestModuleFactoryAdapter):
moduleName = "GPX Parser" moduleName = "GPX Parser"
def getModuleDisplayName(self): def getModuleDisplayName(self):
return self.moduleName return self.moduleName
...@@ -75,158 +79,176 @@ def getModuleDescription(self): ...@@ -75,158 +79,176 @@ def getModuleDescription(self):
def getModuleVersionNumber(self): def getModuleVersionNumber(self):
return "1.2" return "1.2"
def isDataSourceIngestModuleFactory(self): def isFileIngestModuleFactory(self):
return True return True
def createDataSourceIngestModule(self, ingestOptions): def createFileIngestModule(self, ingestOptions):
return GPXParserDataSourceIngestModule() return GPXParserFileIngestModule()
# Data Source-level ingest module. One gets created per data source.
class GPXParserDataSourceIngestModule(DataSourceIngestModule):
logger = Logger.getLogger(GPXParserDataSourceIngestModuleFactory.moduleName) # File level ingest module.
class GPXParserFileIngestModule(FileIngestModule):
logger = Logger.getLogger(
GPXParserFileIngestModuleFactory.moduleName)
writeDebugMsgs = False writeDebugMsgs = False
def log(self, level, msg): def log(self, level, msg):
self.logger.logp(level, self.__class__.__name__, inspect.stack()[1][3], msg) self.logger.logp(level, self.__class__.__name__,
inspect.stack()[1][3], msg)
def __init__(self): def __init__(self):
self.context = None self.context = None
self.fileCount = 0
# Where any setup and configuration is done. # Get the module name, it will be needed for adding attributes
def startUp(self, context): self.moduleName = GPXParserFileIngestModuleFactory.moduleName
self.context = context
# Where the analysis is done.
def process(self, dataSource, progressBar):
# We don't know how much work there is yet.
progressBar.switchToIndeterminate()
# Get the case database and its blackboard. # Get the case database and its blackboard.
skCase = Case.getCurrentCase().getSleuthkitCase() self.skCase = Case.getCurrentCase().getSleuthkitCase()
blackboard = skCase.getBlackboard() self.blackboard = self.skCase.getBlackboard()
# Get any files with a .gpx extension.
# It would perhaps be better to get these files by MIME type instead.
# RC: It would also be better if this were a file level ingest module so it could process files extracted from archives.
fileManager = Case.getCurrentCase().getServices().getFileManager()
files = fileManager.findFiles(dataSource, "%.gpx")
# Update the progress bar now that we know how much work there is to do.
numFiles = len(files)
if self.writeDebugMsgs: self.log(Level.INFO, "Found " + str(numFiles) + " GPX files")
progressBar.switchToDeterminate(numFiles)
# Get the module name, it will be needed for adding attributes
moduleName = GPXParserDataSourceIngestModuleFactory.moduleName
# Check if a folder for this module is present in the case Temp directory. # Check if a folder for this module is present in the case Temp directory.
# If not, create it. # If not, create it.
dirName = os.path.join(Case.getCurrentCase().getTempDirectory(), "GPX_Parser_Module") self.dirName = os.path.join(
Case.getCurrentCase().getTempDirectory(), "GPX_Parser_Module")
try: try:
os.stat(dirName) os.stat(self.dirName)
except: except:
os.mkdir(dirName) os.mkdir(self.dirName)
# Where any setup and configuration is done.
def startUp(self, context):
self.context = context
self.fileCount = 0
# Where the file analysis is done.
def process(self, file):
if not file.getName().lower().endswith(".gpx"):
return IngestModule.ProcessResult.OK
# Create a temp file name. It appears that we cannot close and delete # Create a temp file name. It appears that we cannot close and delete
# this file, but we can overwrite it for each file we need to process. # this file, but we can overwrite it for each file we need to process.
fileName = os.path.join(dirName, "tmp.gpx") fileName = os.path.join(self.dirName, uuid.uuid4().hex + ".gpx")
fileCount = 0;
for file in files:
# Create a GeoArtifactsHelper for this file.
geoArtifactHelper = GeoArtifactsHelper(skCase, moduleName, None, file)
# Check if the user pressed cancel while we were busy.
if self.context.isJobCancelled():
return IngestModule.ProcessResult.OK
if self.writeDebugMsgs: self.log(Level.INFO, "Processing " + file.getUniquePath() + " (objID = " + str(file.getId()) + ")")
fileCount += 1
# Write the file so that it can be parsed by gpxpy.
localFile = File(fileName)
ContentUtils.writeToFile(file, localFile)
# Send the file to gpxpy for parsing.
gpxfile = open(fileName)
try:
gpx = gpxpy.parse(gpxfile)
if self.writeDebugMsgs: self.log(Level.INFO, "Parsed " + file.getUniquePath() + " (objID = " + str(file.getId()) + ")")
except Exception as e:
self.log(Level.WARNING, "Error parsing file " + file.getUniquePath() + " (objID = " + str(file.getId()) + "):" + str(e))
continue
if gpx:
if self.writeDebugMsgs: self.log(Level.INFO, "Processing tracks from " + file.getUniquePath() + " (objID = " + str(file.getId()) + ")")
for track in gpx.tracks:
for segment in track.segments:
geoPointList = GeoTrackPoints()
for point in segment.points:
elevation = 0
if point.elevation != None:
elevation = point.elevation
timeStamp = 0
try:
if (point.time != None):
timeStamp = long(time.mktime(point.time.timetuple()))
except Exception as e:
self.log(Level.WARNING, "Error getting track timestamp from " + file.getUniquePath() + " (objID = " + str(file.getId()) + "):" + str(e))
geoPointList.addPoint(TrackPoint(point.latitude, point.longitude, elevation, None, 0, 0, 0, timeStamp))
try:
geoArtifactHelper.addTrack("Track", geoPointList, None)
except Blackboard.BlackboardException as e:
self.log(Level.SEVERE, "Error posting GPS track artifact for " + file.getUniquePath() + " (objID = " + str(file.getId()) + "):" + e.getMessage())
except TskCoreException as e:
self.log(Level.SEVERE, "Error creating GPS track artifact for " + file.getUniquePath() + " (objID = " + str(file.getId()) + "):" + e.getMessage())
if self.writeDebugMsgs: self.log(Level.INFO, "Processing waypoints from " + file.getUniquePath() + " (objID = " + str(file.getId()) + ")")
for waypoint in gpx.waypoints:
try:
art = file.newArtifact(BlackboardArtifact.ARTIFACT_TYPE.TSK_GPS_BOOKMARK)
attributes = ArrayList() # Create a GeoArtifactsHelper for this file.
attributes.add(BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_GEO_LATITUDE.getTypeID(), moduleName, waypoint.latitude)) geoArtifactHelper = GeoArtifactsHelper(
attributes.add(BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_GEO_LONGITUDE.getTypeID(), moduleName, waypoint.longitude)) self.skCase, self.moduleName, None, file)
attributes.add(BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_FLAG.getTypeID(), moduleName, "Waypoint"))
attributes.add(BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME.getTypeID(), moduleName, waypoint.name))
attributes.add(BlackboardAttribute(BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PROG_NAME.getTypeID(), moduleName, "GPXParser"))
art.addAttributes(attributes)
blackboard.postArtifact(art, moduleName) if self.writeDebugMsgs:
self.log(Level.INFO, "Processing " + file.getUniquePath() +
" (objID = " + str(file.getId()) + ")")
except Blackboard.BlackboardException as e: # Write the file so that it can be parsed by gpxpy.
self.log(Level.SEVERE, "Error posting GPS bookmark artifact for " + file.getUniquePath() + " (objID = " + str(file.getId()) + "):" + e.getMessage()) localFile = File(fileName)
except TskCoreException as e: ContentUtils.writeToFile(file, localFile)
self.log(Level.SEVERE, "Error creating GPS bookmark artifact for " + file.getUniquePath() + " (objID = " + str(file.getId()) + "):" + e.getMessage())
if self.writeDebugMsgs: self.log(Level.INFO, "Processing routes from " + file.getUniquePath() + " (objID = " + str(file.getId()) + ")") # Send the file to gpxpy for parsing.
for route in gpx.routes: gpxfile = open(fileName)
try:
gpx = gpxpy.parse(gpxfile)
if self.writeDebugMsgs:
self.log(Level.INFO, "Parsed " + file.getUniquePath() +
" (objID = " + str(file.getId()) + ")")
except Exception as e:
self.log(Level.WARNING, "Error parsing file " + file.getUniquePath() +
" (objID = " + str(file.getId()) + "):" + str(e))
return IngestModule.ProcessResult.ERROR
if gpx:
if self.writeDebugMsgs:
self.log(Level.INFO, "Processing tracks from " +
file.getUniquePath() + " (objID = " + str(file.getId()) + ")")
for track in gpx.tracks:
for segment in track.segments:
geoPointList = GeoTrackPoints()
for point in segment.points:
elevation = 0
if point.elevation != None:
elevation = point.elevation
timeStamp = 0
try:
if (point.time != None):
timeStamp = long(time.mktime(
point.time.timetuple()))
except Exception as e:
self.log(Level.WARNING, "Error getting track timestamp from " +
file.getUniquePath() + " (objID = " + str(file.getId()) + "):" + str(e))
geoWaypoints = GeoWaypoints() geoPointList.addPoint(TrackPoint(
point.latitude, point.longitude, elevation, None, 0, 0, 0, timeStamp))
for point in route.points:
geoWaypoints.addPoint(Waypoint(point.latitude, point.longitude, point.elevation, point.name))
try: try:
geoArtifactHelper.addRoute(None, None, geoWaypoints, None) geoArtifactHelper.addTrack("Track", geoPointList, None)
except Blackboard.BlackboardException as e: except Blackboard.BlackboardException as e:
self.log("Error posting GPS route artifact for " + file.getUniquePath() + " (objID = " + str(file.getId()) + "):" + e.getMessage()) self.log(Level.SEVERE, "Error posting GPS track artifact for " +
file.getUniquePath() + " (objID = " + str(file.getId()) + "):" + e.getMessage())
except TskCoreException as e: except TskCoreException as e:
self.log(Level.SEVERE, "Error creating GPS route artifact for " + file.getUniquePath() + " (objID = " + str(file.getId()) + "):" + e.getMessage()) self.log(Level.SEVERE, "Error creating GPS track artifact for " +
file.getUniquePath() + " (objID = " + str(file.getId()) + "):" + e.getMessage())
# Update the progress bar.
progressBar.progress(fileCount) if self.writeDebugMsgs:
self.log(Level.INFO, "Processing waypoints from " +
# Post a message to the ingest messages inbox. file.getUniquePath() + " (objID = " + str(file.getId()) + ")")
message = IngestMessage.createMessage(IngestMessage.MessageType.DATA, moduleName, "Processed %d files" % fileCount)
IngestServices.getInstance().postMessage(message) for waypoint in gpx.waypoints:
return IngestModule.ProcessResult.OK;
try:
art = file.newArtifact(
BlackboardArtifact.ARTIFACT_TYPE.TSK_GPS_BOOKMARK)
attributes = ArrayList()
attributes.add(BlackboardAttribute(
BlackboardAttribute.ATTRIBUTE_TYPE.TSK_GEO_LATITUDE.getTypeID(), self.moduleName, waypoint.latitude))
attributes.add(BlackboardAttribute(
BlackboardAttribute.ATTRIBUTE_TYPE.TSK_GEO_LONGITUDE.getTypeID(), self.moduleName, waypoint.longitude))
attributes.add(BlackboardAttribute(
BlackboardAttribute.ATTRIBUTE_TYPE.TSK_FLAG.getTypeID(), self.moduleName, "Waypoint"))
attributes.add(BlackboardAttribute(
BlackboardAttribute.ATTRIBUTE_TYPE.TSK_NAME.getTypeID(), self.moduleName, waypoint.name))
attributes.add(BlackboardAttribute(
BlackboardAttribute.ATTRIBUTE_TYPE.TSK_PROG_NAME.getTypeID(), self.moduleName, "GPXParser"))
art.addAttributes(attributes)
self.blackboard.postArtifact(art, self.moduleName)
except Blackboard.BlackboardException as e:
self.log(Level.SEVERE, "Error posting GPS bookmark artifact for " +
file.getUniquePath() + " (objID = " + str(file.getId()) + "):" + e.getMessage())
except TskCoreException as e:
self.log(Level.SEVERE, "Error creating GPS bookmark artifact for " +
file.getUniquePath() + " (objID = " + str(file.getId()) + "):" + e.getMessage())
if self.writeDebugMsgs:
self.log(Level.INFO, "Processing routes from " +
file.getUniquePath() + " (objID = " + str(file.getId()) + ")")
for route in gpx.routes:
geoWaypoints = GeoWaypoints()
for point in route.points:
geoWaypoints.addPoint(
Waypoint(point.latitude, point.longitude, point.elevation, point.name))
try:
geoArtifactHelper.addRoute(None, None, geoWaypoints, None)
except Blackboard.BlackboardException as e:
self.log("Error posting GPS route artifact for " + file.getUniquePath() +
" (objID = " + str(file.getId()) + "):" + e.getMessage())
except TskCoreException as e:
self.log(Level.SEVERE, "Error creating GPS route artifact for " +
file.getUniquePath() + " (objID = " + str(file.getId()) + "):" + e.getMessage())
self.fileCount += 1
return IngestModule.ProcessResult.OK
def shutDown(self):
message = IngestMessage.createMessage(
IngestMessage.MessageType.DATA, GPXParserFileIngestModuleFactory.moduleName,
str(self.fileCount) + " files found")
ingestServices = IngestServices.getInstance().postMessage(message)
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment