diff --git a/Core/src/org/sleuthkit/autopsy/datamodel/BlackboardArtifactNode.java b/Core/src/org/sleuthkit/autopsy/datamodel/BlackboardArtifactNode.java index 3b77c81a7ed853b75cda132ea6328c46f02dbf83..6166fdd4d644df698ee2a5cbaa819ac7ffd18d92 100644 --- a/Core/src/org/sleuthkit/autopsy/datamodel/BlackboardArtifactNode.java +++ b/Core/src/org/sleuthkit/autopsy/datamodel/BlackboardArtifactNode.java @@ -126,6 +126,36 @@ protected Sheet createSheet() { } final int artifactTypeId = artifact.getArtifactTypeID(); + // If mismatch, add props for extension and file type + if (artifactTypeId == BlackboardArtifact.ARTIFACT_TYPE.TSK_EXT_MISMATCH_DETECTED.getTypeID()) { + String actualExt = ""; + int i = associated.getName().lastIndexOf("."); + if ((i > -1) && ((i + 1) < associated.getName().length())) { + actualExt = associated.getName().substring(i + 1).toLowerCase(); + } + ss.put(new NodeProperty("Extension", "Extension", NO_DESCR, actualExt)); + + try { + String actualMimeType = ""; + ArrayList<BlackboardArtifact> artList = associated.getAllArtifacts(); + for (BlackboardArtifact art : artList) { + List<BlackboardAttribute> atrList = art.getAttributes(); + for (BlackboardAttribute att : atrList) { + if (att.getAttributeTypeID() == BlackboardAttribute.ATTRIBUTE_TYPE.TSK_FILE_TYPE_SIG.getTypeID()) { + actualMimeType = att.getValueString(); + } + } + } + if (actualMimeType.isEmpty()) { + logger.log(Level.WARNING, "Could not find expected TSK_FILE_TYPE_SIG attribute."); + } else { + ss.put(new NodeProperty("MIME Type", "MIME Type", NO_DESCR, actualMimeType)); + } + } catch (TskCoreException ex) { + logger.log(Level.WARNING, "Error while searching for TSK_FILE_TYPE_SIG attribute: ", ex); + } + } + if (Arrays.asList(SHOW_UNIQUE_PATH).contains(artifactTypeId)) { String sourcePath = ""; try { diff --git a/Core/src/org/sleuthkit/autopsy/ingest/GeneralIngestConfigurator.java b/Core/src/org/sleuthkit/autopsy/ingest/GeneralIngestConfigurator.java index eaeb0eefc64fde18d60952b90a000dbec3e7de46..a9769a5b5b959f96accc68b49faefbe2b12d244d 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/GeneralIngestConfigurator.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/GeneralIngestConfigurator.java @@ -19,6 +19,7 @@ package org.sleuthkit.autopsy.ingest; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import javax.swing.JPanel; import org.openide.util.lookup.ServiceProvider; @@ -29,6 +30,7 @@ public class GeneralIngestConfigurator implements IngestConfigurator { public static final String ENABLED_INGEST_MODULES_KEY = "Enabled_Ingest_Modules"; + public static final String DISABLED_INGEST_MODULES_KEY = "Disabled_Ingest_Modules"; public static final String PARSE_UNALLOC_SPACE_KEY = "Process_Unallocated_Space"; private List<Content> contentToIngest; private IngestManager manager; @@ -51,20 +53,57 @@ public List<String> setContext(String context) { private List<String> loadSettingsForContext() { List<String> messages = new ArrayList<>(); + List<IngestModuleAbstract> allModules = IngestManager.getDefault().enumerateAllModules(); // If there is no enabled ingest modules setting for this user, default to enabling all // of the ingest modules the IngestManager has loaded. if (ModuleSettings.settingExists(moduleContext, ENABLED_INGEST_MODULES_KEY) == false) { - String defaultSetting = moduleListToCsv(IngestManager.getDefault().enumerateAllModules()); + String defaultSetting = moduleListToCsv(allModules); ModuleSettings.setConfigSetting(moduleContext, ENABLED_INGEST_MODULES_KEY, defaultSetting); } + String[] enabledModuleNames = ModuleSettings.getConfigSetting(moduleContext, ENABLED_INGEST_MODULES_KEY).split(", "); + ArrayList<String> enabledList = new ArrayList<>(Arrays.asList(enabledModuleNames)); + + // Check for modules that are missing from the config file + + String[] disabledModuleNames = null; + // Older config files won't have the disabled list, so don't assume it exists + if (ModuleSettings.settingExists(moduleContext, DISABLED_INGEST_MODULES_KEY)) { + disabledModuleNames = ModuleSettings.getConfigSetting(moduleContext, DISABLED_INGEST_MODULES_KEY).split(", "); + } + + for (IngestModuleAbstract module : allModules) { + boolean found = false; + + // Check enabled first + for (String moduleName : enabledModuleNames) { + if (module.getName().equals(moduleName)) { + found = true; + break; + } + } + + // Then check disabled + if (!found && (disabledModuleNames != null)) { + for (String moduleName : disabledModuleNames) { + if (module.getName().equals(moduleName)) { + found = true; + break; + } + } + } + + if (!found) { + enabledList.add(module.getName()); + // It will get saved to file later + } + } + // Get the enabled ingest modules setting, check for missing modules, and pass the setting to // the UI component. - List<IngestModuleAbstract> allModules = IngestManager.getDefault().enumerateAllModules(); - String[] enabledModuleNames = ModuleSettings.getConfigSetting(moduleContext, ENABLED_INGEST_MODULES_KEY).split(", "); List<IngestModuleAbstract> enabledModules = new ArrayList<>(); - for (String moduleName : enabledModuleNames) { + for (String moduleName : enabledList) { if (moduleName.equals("Thunderbird Parser") || moduleName.equals("MBox Parser")) { moduleName = "Email Parser"; @@ -112,6 +151,10 @@ public void save() { String enabledModulesCsvList = moduleListToCsv(ingestDialogPanel.getModulesToStart()); ModuleSettings.setConfigSetting(moduleContext, ENABLED_INGEST_MODULES_KEY, enabledModulesCsvList); + // Save the user's configuration of the set of disabled ingest modules. + String disabledModulesCsvList = moduleListToCsv(ingestDialogPanel.getDisabledModules()); + ModuleSettings.setConfigSetting(moduleContext, DISABLED_INGEST_MODULES_KEY, disabledModulesCsvList); + // Save the user's setting for the process unallocated space flag. String processUnalloc = Boolean.toString(ingestDialogPanel.processUnallocSpaceEnabled()); ModuleSettings.setConfigSetting(moduleContext, PARSE_UNALLOC_SPACE_KEY, processUnalloc); diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestDialogPanel.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestDialogPanel.java index 331c727576fdac8537b9443c9baf6fff9c302540..198f19da134362ad0fc8a849736613cb6bfb4aeb 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestDialogPanel.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestDialogPanel.java @@ -60,7 +60,7 @@ public void setContext(String context) { this.context = context; } - + public IngestModuleAbstract getCurrentIngestModule() { return currentModule; } @@ -69,6 +69,10 @@ public List<IngestModuleAbstract> getModulesToStart() { return tableModel.getSelectedModules(); } + public List<IngestModuleAbstract> getDisabledModules() { + return tableModel.getUnSelectedModules(); + } + public boolean processUnallocSpaceEnabled() { return processUnallocCheckbox.isSelected(); } @@ -350,6 +354,16 @@ public List<IngestModuleAbstract> getSelectedModules() { return selectedModules; } + public List<IngestModuleAbstract> getUnSelectedModules() { + List<IngestModuleAbstract> unselectedModules = new ArrayList<>(); + for (Map.Entry<IngestModuleAbstract, Boolean> entry : moduleData) { + if (!entry.getValue().booleanValue()) { + unselectedModules.add(entry.getKey()); + } + } + return unselectedModules; + } + /** * Sets the given modules as selected in the modules table * @param selectedModules diff --git a/FileTypeId/nbproject/project.xml b/FileTypeId/nbproject/project.xml index bcf4cb30092f7e69fac8ed6871ca5053486f8c91..8d3ea957d30536719e253cf50cae8f6618dd1f95 100644 --- a/FileTypeId/nbproject/project.xml +++ b/FileTypeId/nbproject/project.xml @@ -91,7 +91,9 @@ </run-dependency> </dependency> </module-dependencies> - <public-packages/> + <public-packages> + <package>org.sleuthkit.autopsy.filetypeid</package> + </public-packages> <class-path-extension> <runtime-relative-path>ext/tika-core-1.2.jar</runtime-relative-path> <binary-origin>release/modules/ext/tika-core-1.2.jar</binary-origin> diff --git a/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeDetectionInterface.java b/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeDetectionInterface.java index a05091c1393fdb5afd418a0fa251bd326933cbf2..47f4feb77bbe59c520af40994d912c157c55b0a2 100644 --- a/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeDetectionInterface.java +++ b/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeDetectionInterface.java @@ -36,5 +36,7 @@ public class FileIdInfo { } // You only have one job - FileIdInfo attemptMatch(AbstractFile abstractFile); + FileIdInfo attemptMatch(AbstractFile abstractFile); + + boolean isMimeTypeDetectable(String mimeType); } diff --git a/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdIngestModule.java b/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdIngestModule.java index 81c623d9b36330153f6b768f4620e1e781bc2853..39878ca1d6eda807ead6c9182cea9d419b499b7f 100644 --- a/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdIngestModule.java +++ b/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdIngestModule.java @@ -48,7 +48,7 @@ public class FileTypeIdIngestModule extends org.sleuthkit.autopsy.ingest.IngestM private static long matchTime = 0; private static int messageId = 0; private static long numFiles = 0; - private static boolean skipKnown = false; + private static boolean skipKnown = true; private FileTypeIdSimpleConfigPanel simpleConfigPanel; private IngestServices services; @@ -173,4 +173,15 @@ public boolean hasBackgroundJobsRunning() { public static void setSkipKnown(boolean flag) { skipKnown = flag; } + + /** + * Validate if a given mime type is in the detector's registry. + * @param mimeType Full string of mime type, e.g. "text/html" + * @return true if detectable + */ + public static boolean isMimeTypeDetectable(String mimeType) { + FileTypeDetectionInterface detector = new TikaFileTypeDetector(); + return detector.isMimeTypeDetectable(mimeType); + } + } \ No newline at end of file diff --git a/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdSimpleConfigPanel.form b/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdSimpleConfigPanel.form index fe15876df1f2f462bddd247fa2e46a26f56b44aa..29004307b52a1ed983858091285152579a9ff659 100644 --- a/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdSimpleConfigPanel.form +++ b/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdSimpleConfigPanel.form @@ -36,6 +36,7 @@ <SubComponents> <Component class="javax.swing.JCheckBox" name="skipKnownCheckBox"> <Properties> + <Property name="selected" type="boolean" value="true"/> <Property name="text" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor"> <ResourceString bundle="org/sleuthkit/autopsy/filetypeid/Bundle.properties" key="FileTypeIdSimpleConfigPanel.skipKnownCheckBox.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, "{key}")"/> </Property> diff --git a/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdSimpleConfigPanel.java b/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdSimpleConfigPanel.java index 5a3f11eda9ee0334175ea5cfbd29280b781b8dd5..43576d81844fce017c6ef60dc39e51fa4332d6f3 100644 --- a/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdSimpleConfigPanel.java +++ b/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/FileTypeIdSimpleConfigPanel.java @@ -44,6 +44,7 @@ private void initComponents() { skipKnownCheckBox = new javax.swing.JCheckBox(); + skipKnownCheckBox.setSelected(true); skipKnownCheckBox.setText(org.openide.util.NbBundle.getMessage(FileTypeIdSimpleConfigPanel.class, "FileTypeIdSimpleConfigPanel.skipKnownCheckBox.text")); // NOI18N skipKnownCheckBox.setToolTipText(org.openide.util.NbBundle.getMessage(FileTypeIdSimpleConfigPanel.class, "FileTypeIdSimpleConfigPanel.skipKnownCheckBox.toolTipText")); // NOI18N skipKnownCheckBox.addActionListener(new java.awt.event.ActionListener() { diff --git a/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/TikaFileTypeDetector.java b/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/TikaFileTypeDetector.java index 365584db7fd64c375b65341c8e7561cefd030f5b..ff24f485e5ed47f26e67a199c1377ec2b78e5ea4 100644 --- a/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/TikaFileTypeDetector.java +++ b/FileTypeId/src/org/sleuthkit/autopsy/filetypeid/TikaFileTypeDetector.java @@ -18,8 +18,11 @@ */ package org.sleuthkit.autopsy.filetypeid; +import java.util.SortedSet; import org.openide.util.Exceptions; import org.apache.tika.Tika; +import org.apache.tika.mime.MediaType; +import org.apache.tika.mime.MimeTypes; import org.sleuthkit.datamodel.AbstractFile; @@ -52,4 +55,27 @@ public FileTypeDetectionInterface.FileIdInfo attemptMatch(AbstractFile abstractF } } + /** + * Validate if a given mime type is in the registry. + * For Tika, we remove the string "tika" from all MIME names, + * e.g. use "application/x-msoffice" NOT "application/x-tika-msoffice" + * @param mimeType Full string of mime type, e.g. "text/html" + * @return true if detectable + */ + @Override + public boolean isMimeTypeDetectable(String mimeType) { + boolean ret = false; + + SortedSet<MediaType> m = MimeTypes.getDefaultMimeTypes().getMediaTypeRegistry().getTypes(); + String[] split = mimeType.split("/"); + + if (split.length == 2) { + String type = split[0]; + String subtype = split[1]; + MediaType mediaType = new MediaType(type, subtype); + ret = m.contains(mediaType); + } + + return ret; + } } diff --git a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbIngestModule.java b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbIngestModule.java index f3f63ebd16df227e777259d054574d040fda382b..6e8d895f06e46dca48c1d7431f278149b26cc5e9 100644 --- a/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbIngestModule.java +++ b/HashDatabase/src/org/sleuthkit/autopsy/hashdatabase/HashDbIngestModule.java @@ -50,6 +50,7 @@ public class HashDbIngestModule extends IngestModuleAbstractFile { public final static String MODULE_DESCRIPTION = "Identifies known and notables files using supplied hash databases, such as a standard NSRL database."; final public static String MODULE_VERSION = Version.getVersion(); private static final Logger logger = Logger.getLogger(HashDbIngestModule.class.getName()); + private static final int MAX_COMMENT_SIZE = 500; private HashDbSimpleConfigPanel simpleConfigPanel; private HashDbConfigPanel advancedConfigPanel; private IngestServices services; @@ -227,9 +228,24 @@ private ProcessResult processFile(AbstractFile file) { services.postMessage(IngestMessage.createErrorMessage(++messageId, HashDbIngestModule.this, "Hash Lookup Error: " + name, "Error encountered while setting known bad state for " + name + ".")); ret = ProcessResult.ERROR; - } + } String hashSetName = db.getHashSetName(); - postHashSetHitToBlackboard(file, md5Hash, hashSetName, db.getSendIngestMessages()); + + String comment = ""; + ArrayList<String> comments = db.lookUp(file).getComments(); + int i = 0; + for (String c : comments) { + comment += c; + if (++i > 1) { + c += ". "; + } + if (comment.length() > MAX_COMMENT_SIZE) { + comment = comment.substring(0, MAX_COMMENT_SIZE) + "..."; + break; + } + } + + postHashSetHitToBlackboard(file, md5Hash, hashSetName, comment, db.getSendIngestMessages()); } lookuptime += (System.currentTimeMillis() - lookupstart); } catch (TskException ex) { @@ -271,7 +287,7 @@ private ProcessResult processFile(AbstractFile file) { return ret; } - private void postHashSetHitToBlackboard(AbstractFile abstractFile, String md5Hash, String hashSetName, boolean showInboxMessage) { + private void postHashSetHitToBlackboard(AbstractFile abstractFile, String md5Hash, String hashSetName, String comment, boolean showInboxMessage) { try { BlackboardArtifact badFile = abstractFile.newArtifact(ARTIFACT_TYPE.TSK_HASHSET_HIT); //TODO Revisit usage of deprecated constructor as per TSK-583 @@ -280,6 +296,9 @@ private void postHashSetHitToBlackboard(AbstractFile abstractFile, String md5Has badFile.addAttribute(att2); BlackboardAttribute att3 = new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_HASH_MD5.getTypeID(), MODULE_NAME, md5Hash); badFile.addAttribute(att3); + BlackboardAttribute att4 = new BlackboardAttribute(ATTRIBUTE_TYPE.TSK_COMMENT.getTypeID(), MODULE_NAME, comment); + badFile.addAttribute(att4); + if (showInboxMessage) { StringBuilder detailsSb = new StringBuilder(); //details diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java index 592e363c28d465c937685a8adb4da6d38afc30a6..5978c64bef6cf3f3d9745b9a3ec753e6c7b8d51e 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011-2013 Basis Technology Corp. + * Copyright 2011-2014 Basis Technology Corp. * Contact: carrier <at> sleuthkit <dot> org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -22,7 +22,6 @@ import java.awt.event.ActionListener; import java.io.IOException; import java.io.InputStream; -import java.lang.Long; import java.util.ArrayList; import java.util.Collection; import java.util.HashMap; @@ -128,7 +127,8 @@ int getTime() { private static List<AbstractFileExtract> textExtractors; private static AbstractFileStringExtract stringExtractor; private boolean initialized = false; - private KeywordSearchConfigurationPanel panel; + private KeywordSearchIngestSimplePanel simpleConfigPanel; + private KeywordSearchConfigurationPanel advancedConfigPanel; private Tika tikaFormatDetector; @@ -436,26 +436,36 @@ public boolean hasAdvancedConfiguration() { @Override public javax.swing.JPanel getSimpleConfiguration(String context) { KeywordSearchListsXML.getCurrent().reload(); - return new KeywordSearchIngestSimplePanel(); + + if (null == simpleConfigPanel) { + simpleConfigPanel = new KeywordSearchIngestSimplePanel(); + } + else { + simpleConfigPanel.load(); + } + + return simpleConfigPanel; } @Override public javax.swing.JPanel getAdvancedConfiguration(String context) { - //return KeywordSearchConfigurationPanel.getDefault(); - getPanel().load(); - return getPanel(); - } - - private KeywordSearchConfigurationPanel getPanel() { - if (panel == null) { - panel = new KeywordSearchConfigurationPanel(); + if (advancedConfigPanel == null) { + advancedConfigPanel = new KeywordSearchConfigurationPanel(); } - return panel; + + advancedConfigPanel.load(); + return advancedConfigPanel; } @Override public void saveAdvancedConfiguration() { - getPanel().store(); + if (advancedConfigPanel != null) { + advancedConfigPanel.store(); + } + + if (simpleConfigPanel != null) { + simpleConfigPanel.load(); + } } @Override diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestSimplePanel.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestSimplePanel.java index fb1bc964a8d0096030103c41024a1633d0fac37a..7c6b8090d6eff816bc81e41e3f67e3adc3e16d38 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestSimplePanel.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestSimplePanel.java @@ -70,10 +70,12 @@ private void customizeComponents() { reloadEncodings(); } - public void load() { + public void load() { + KeywordSearchListsXML.getCurrent().reload(); reloadLists(); reloadLangs(); reloadEncodings(); + tableModel.fireTableDataChanged(); } public void store() { diff --git a/nbproject/project.properties b/nbproject/project.properties index 5176c4e932e430cd64e9f7358ca5cd9650cdadc5..aea65021564c3fb3388a677994c60a863188c517 100644 --- a/nbproject/project.properties +++ b/nbproject/project.properties @@ -8,6 +8,7 @@ app.version=3.0.8 ### Build type isn't used at this point, but it may be useful ### Must be one of: DEVELOPMENT, RELEASE build.type=RELEASE +project.org.sleuthkit.autopsy.ewfverify=ewfVerify #build.type=DEVELOPMENT update_versions=false #custom JVM options @@ -32,8 +33,8 @@ modules=\ ${project.org.sleuthkit.autopsy.sevenzip}:\ ${project.org.sleuthkit.autopsy.scalpel}:\ ${project.org.sleuthkit.autopsy.timeline}:\ - ${project.org.sleuthkit.autopsy.ewfverify}:\ - ${project.org.sleuthkit.autopsy.filetypeid} + ${project.org.sleuthkit.autopsy.filetypeid}:\ + ${project.org.sleuthkit.autopsy.ewfverify} project.org.sleuthkit.autopsy.core=Core project.org.sleuthkit.autopsy.corelibs=CoreLibs project.org.sleuthkit.autopsy.hashdatabase=HashDatabase @@ -46,5 +47,4 @@ project.org.sleuthkit.autopsy.sevenzip=SevenZip project.org.sleuthkit.autopsy.scalpel=ScalpelCarver project.org.sleuthkit.autopsy.timeline=Timeline project.org.sleuthkit.autopsy.filetypeid=FileTypeId -project.org.sleuthkit.autopsy.ewfverify=ewfVerify