diff --git a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobSettings.java b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobSettings.java index d0cb616e0676f1891ade2cf9aba18a010eacf651..e0133db092cbf38647dce33fbc2206bc54eeef1e 100644 --- a/Core/src/org/sleuthkit/autopsy/ingest/IngestJobSettings.java +++ b/Core/src/org/sleuthkit/autopsy/ingest/IngestJobSettings.java @@ -54,7 +54,7 @@ public class IngestJobSettings { private static final Logger logger = Logger.getLogger(IngestJobSettings.class.getName()); private final String context; private String moduleSettingsFolderPath; - private static final CharSequence pythonModuleSettingsPrefixCS = "org.python.proxies.".subSequence(0, "org.python.proxies.".length() - 1); + private static final CharSequence pythonModulePrefixCS = "org.python.proxies.".subSequence(0, "org.python.proxies.".length() - 1); private final List<IngestModuleTemplate> moduleTemplates; private boolean processUnallocatedSpace; private final List<String> warnings; @@ -292,7 +292,7 @@ private HashSet<String> getModulesNamesFromSetting(String key, String defaultSet * @return True or false */ private boolean isPythonModuleSettingsFile(String moduleSettingsFilePath) { - return moduleSettingsFilePath.contains(pythonModuleSettingsPrefixCS); + return moduleSettingsFilePath.contains(pythonModulePrefixCS); } /** @@ -341,7 +341,11 @@ private IngestModuleIngestJobSettings loadModuleSettings(IngestModuleFactory fac * @return The file path. */ private String getModuleSettingsFilePath(IngestModuleFactory factory) { - String fileName = factory.getClass().getCanonicalName() + IngestJobSettings.MODULE_SETTINGS_FILE_EXT; + String className = factory.getClass().getCanonicalName(); + if (className.contains(pythonModulePrefixCS)) { + className = className.replaceAll("\\$[\\d]$", "\\$"); + } + String fileName = className + IngestJobSettings.MODULE_SETTINGS_FILE_EXT; Path path = Paths.get(this.moduleSettingsFolderPath, fileName); return path.toAbsolutePath().toString(); } diff --git a/Core/src/org/sleuthkit/autopsy/python/Bundle.properties b/Core/src/org/sleuthkit/autopsy/python/Bundle.properties index 9016f7518a66bb7da99224c860b81fba77134265..68a5d63320541f56a94fbbc4845503a03629100c 100755 --- a/Core/src/org/sleuthkit/autopsy/python/Bundle.properties +++ b/Core/src/org/sleuthkit/autopsy/python/Bundle.properties @@ -1,2 +1,4 @@ JythonModuleLoader.errorMessages.failedToOpenModule=Failed to open {0}. See log for details. -JythonModuleLoader.errorMessages.failedToLoadModule=Failed to load {0}. {1}. See log for details. \ No newline at end of file +JythonModuleLoader.errorMessages.failedToLoadModule=Failed to load {0}. {1}. See log for details. +JythonModuleLoader.createObjectFromScript.reloadScript.title=Python Module reloaded. +JythonModuleLoader.createObjectFromScript.reloadScript.msg=Python module {0} has changed since the previous ingest operation. This module is reloaded. \ No newline at end of file diff --git a/Core/src/org/sleuthkit/autopsy/python/JythonModuleLoader.java b/Core/src/org/sleuthkit/autopsy/python/JythonModuleLoader.java index 93bd39d5d98065ffbe8a93a43a6245cb2cb69531..dd476e895d2c2949912a3658bdcc9965f9e53e55 100755 --- a/Core/src/org/sleuthkit/autopsy/python/JythonModuleLoader.java +++ b/Core/src/org/sleuthkit/autopsy/python/JythonModuleLoader.java @@ -19,12 +19,20 @@ package org.sleuthkit.autopsy.python; import java.io.File; +import java.io.FileInputStream; import java.io.FileNotFoundException; +import java.io.FileOutputStream; import java.io.FilenameFilter; +import java.io.IOException; +import java.io.ObjectInputStream; +import java.io.ObjectOutputStream; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Scanner; import java.util.Set; import java.util.logging.Level; @@ -35,6 +43,7 @@ import org.openide.util.NbBundle; import org.python.util.PythonInterpreter; import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; import org.sleuthkit.autopsy.coreutils.PlatformUtil; import org.sleuthkit.autopsy.ingest.IngestModuleFactory; import org.sleuthkit.autopsy.report.GeneralReportModule; @@ -46,6 +55,10 @@ public final class JythonModuleLoader { private static final Logger logger = Logger.getLogger(JythonModuleLoader.class.getName()); + private static final String PYTHON_MODULE_FOLDERS_LIST = Paths.get(PlatformUtil.getUserConfigDirectory(), "IngestModuleSettings", "listOfPythonModules.settings").toAbsolutePath().toString(); + // maintain a private list of loaded modules (folders) and their last 'loading' time. + // Check this list before reloading the modules. + private static Map<String, Long> pythonModuleFolderList; /** * Get ingest module factories implemented using Jython. @@ -72,6 +85,21 @@ private static <T> List<T> getInterfaceImplementations(LineFilter filter, Class< Set<File> pythonModuleDirs = new HashSet<>(); PythonInterpreter interpreter = new PythonInterpreter(); + // deserialize the list of python modules from the disk. + if (new File(PYTHON_MODULE_FOLDERS_LIST).exists()) { + // try deserializing if PYTHON_MODULE_FOLDERS_LIST exists. Else, + // instantiate a new pythonModuleFolderList. + try (FileInputStream fis = new FileInputStream(PYTHON_MODULE_FOLDERS_LIST); ObjectInputStream ois = new ObjectInputStream(fis)) { + pythonModuleFolderList = (HashMap) ois.readObject(); + } catch (IOException | ClassNotFoundException ex) { + logger.log(Level.INFO, "Unable to deserialize pythonModuleList from existing " + PYTHON_MODULE_FOLDERS_LIST + ". New pythonModuleList instantiated", ex); // NON-NLS + pythonModuleFolderList = new HashMap<>(); + } + } else { + pythonModuleFolderList = new HashMap<>(); + logger.log(Level.INFO, "{0} does not exist. New pythonModuleList instantiated", PYTHON_MODULE_FOLDERS_LIST); // NON-NLS + } + // add python modules from 'autospy/build/cluster/InternalPythonModules' folder // which are copied from 'autopsy/*/release/InternalPythonModules' folders. for (File f : InstalledFileLocator.getDefault().locateAll("InternalPythonModules", JythonModuleLoader.class.getPackage().getName(), false)) { @@ -91,7 +119,14 @@ private static <T> List<T> getInterfaceImplementations(LineFilter filter, Class< if (line.startsWith("class ") && filter.accept(line)) { //NON-NLS String className = line.substring(6, line.indexOf("(")); try { - objects.add(createObjectFromScript(interpreter, script, className, interfaceClass)); + // check if ANY file in the module folder has changed. + // Not only .py files. + boolean reloadModule = hasModuleFolderChanged(file.getAbsolutePath(), file.listFiles()); + objects.add(createObjectFromScript(interpreter, script, className, interfaceClass, reloadModule)); + if (reloadModule) { + MessageNotifyUtil.Notify.info(NbBundle.getMessage(JythonModuleLoader.class, "JythonModuleLoader.createObjectFromScript.reloadScript.title"), + NbBundle.getMessage(JythonModuleLoader.class, "JythonModuleLoader.createObjectFromScript.reloadScript.msg", script.getParent())); // NON-NLS + } } catch (Exception ex) { logger.log(Level.SEVERE, String.format("Failed to load %s from %s", className, script.getAbsolutePath()), ex); //NON-NLS // NOTE: using ex.toString() because the current version is always returning null for ex.getMessage(). @@ -108,12 +143,21 @@ private static <T> List<T> getInterfaceImplementations(LineFilter filter, Class< NotifyDescriptor.ERROR_MESSAGE)); } } + pythonModuleFolderList.put(file.getAbsolutePath(), System.currentTimeMillis()); } } + + // serialize the list of python modules to the disk. + try (FileOutputStream fos = new FileOutputStream(PYTHON_MODULE_FOLDERS_LIST); ObjectOutputStream oos = new ObjectOutputStream(fos)) { + oos.writeObject(pythonModuleFolderList); + } catch (IOException ex) { + logger.log(Level.WARNING, "Error serializing pythonModuleList to the disk.", ex); // NON-NLS + } + return objects; } - private static <T> T createObjectFromScript(PythonInterpreter interpreter, File script, String className, Class<T> interfaceClass) { + private static <T> T createObjectFromScript(PythonInterpreter interpreter, File script, String className, Class<T> interfaceClass, boolean reloadModule) { // Add the directory where the Python script resides to the Python // module search path to allow the script to use other scripts bundled // with it. @@ -124,7 +168,9 @@ private static <T> T createObjectFromScript(PythonInterpreter interpreter, File // reload the module so that the changes made to it can be loaded. interpreter.exec("import " + moduleName); //NON-NLS - interpreter.exec("reload(" + moduleName + ")"); //NON-NLS + if (reloadModule) { + interpreter.exec("reload(" + moduleName + ")"); //NON-NLS + } // Importing the appropriate class from the Py Script which contains multiple classes. interpreter.exec("from " + moduleName + " import " + className); @@ -139,6 +185,20 @@ private static <T> T createObjectFromScript(PythonInterpreter interpreter, File return obj; } + + // returns true if any file inside the Python module folder has been + // modified since last loading of ingest factories. + private static boolean hasModuleFolderChanged(String moduleFolderName, File[] files) { + if (pythonModuleFolderList.containsKey(moduleFolderName)) { + for (File file : files) { + if (file.lastModified() > pythonModuleFolderList.get(moduleFolderName)) { + return true; + } + } + } + return false; + } + private static class PythonScriptFileFilter implements FilenameFilter { @Override diff --git a/docs/doxygen/modDevPython.dox b/docs/doxygen/modDevPython.dox index 9b1b74501f29147b1c78c5e7853da5af8d3c123f..7c3baa3f67a2f4041b30f2449f69c6af06d6edee 100755 --- a/docs/doxygen/modDevPython.dox +++ b/docs/doxygen/modDevPython.dox @@ -67,6 +67,7 @@ This section lists some helpful tips that we have found. These are all now in t - We haven't found a good way to debug while running inside of Autopsy. So, logging becomes critical. You need to go through a bunch of steps to get the logger to display your module name. See the sample module for a log() method that does all of this for you. - When you name the file with your Python module in it, restrict its name to letters, numbers, and underscore (_). - Python modules using external libraries which load native code (SciPy, NumPy, etc.) are currently NOT supported. RuntimeError will be thrown. +- Settings previously serialized to the disk will not persist if any changes are made to the Python module since then; default settings will be loaded. \section mod_dev_py_distribute Distribution