diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AdHocSearchPanel.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AdHocSearchPanel.java index ce6c23f57d40f6efe36c176332c628620c87df0c..c9aa061400ccccaccfc1a3b8115dca3d83fb2a6e 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AdHocSearchPanel.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/AdHocSearchPanel.java @@ -112,9 +112,9 @@ public void search(boolean saveResults) { } if (filesIndexed == 0) { if (isIngestRunning) { - // ELTODO this message should be dependent on whether Solr indexing is enabled or not KeywordSearchUtil.displayDialog(keywordSearchErrorDialogHeader, NbBundle.getMessage(this.getClass(), - "AbstractKeywordSearchPerformer.search.noFilesInIdxMsg"), KeywordSearchUtil.DIALOG_MESSAGE_TYPE.ERROR); + "AbstractKeywordSearchPerformer.search.noFilesInIdxMsg", + KeywordSearchSettings.getUpdateFrequency().getTime()), KeywordSearchUtil.DIALOG_MESSAGE_TYPE.ERROR); } else { KeywordSearchUtil.displayDialog(keywordSearchErrorDialogHeader, NbBundle.getMessage(this.getClass(), "AbstractKeywordSearchPerformer.search.noFilesIdxdMsg"), KeywordSearchUtil.DIALOG_MESSAGE_TYPE.ERROR); diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties index e34155fcd537c7c446b904b76c0219100148f017..42e1ac1dc9c8d92539ea31a66ca5df86ea6cf735 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties @@ -39,8 +39,8 @@ AbstractKeywordSearchPerformer.search.invalidSyntaxHeader=Invalid query statemen AbstractKeywordSearchPerformer.search.searchIngestInProgressTitle=Keyword Search Ingest in Progress AbstractKeywordSearchPerformer.search.ingestInProgressBody=<html>Keyword Search Ingest is currently running.<br />Not all files have been indexed and this search might yield incomplete results.<br />Do you want to proceed with this search anyway?</html> AbstractKeywordSearchPerformer.search.emptyKeywordErrorBody=Keyword list is empty, please add at least one keyword to the list -AbstractKeywordSearchPerformer.search.noFilesInIdxMsg=<html>No files are in index yet. <br />If Solr keyword search indexing was enabled, wait for ingest to complete</html> -AbstractKeywordSearchPerformer.search.noFilesIdxdMsg=<html>No files were indexed.<br />Re-ingest the image with the Keyword Search Module and Solr indexing enabled. </html> +AbstractKeywordSearchPerformer.search.noFilesInIdxMsg=<html>No files are in index yet. <br />Try again later. Index is updated every {0} minutes.</html> +AbstractKeywordSearchPerformer.search.noFilesIdxdMsg=<html>No files were indexed.<br />Re-ingest the image with the Keyword Search Module enabled. </html> ExtractedContentViewer.toolTip=Displays extracted text from files and keyword-search results. Requires Keyword Search ingest to be run on a file to activate this viewer. ExtractedContentViewer.getTitle=Indexed Text HighlightedMatchesSource.toString=Search Results @@ -211,6 +211,11 @@ KeywordSearchGlobalLanguageSettingsPanel.enableUTF8Checkbox.text=Enable UTF8 tex KeywordSearchGlobalLanguageSettingsPanel.ingestSettingsLabel.text=Ingest settings for string extraction from unknown file types (changes effective on next ingest): KeywordSearchGlobalLanguageSettingsPanel.enableUTF16Checkbox.text=Enable UTF16LE and UTF16BE string extraction KeywordSearchGlobalLanguageSettingsPanel.languagesLabel.text=Enabled scripts (languages): +KeywordSearchGlobalSearchSettingsPanel.timeRadioButton1.toolTipText=20 mins. (fastest ingest time) +KeywordSearchGlobalSearchSettingsPanel.timeRadioButton1.text=20 minutes (slowest feedback, fastest ingest) +KeywordSearchGlobalSearchSettingsPanel.timeRadioButton2.toolTipText=10 minutes (faster overall ingest time than default) +KeywordSearchGlobalSearchSettingsPanel.timeRadioButton2.text=10 minutes (slower feedback, faster ingest) +KeywordSearchGlobalSearchSettingsPanel.frequencyLabel.text=Results update frequency during ingest: KeywordSearchGlobalSearchSettingsPanel.skipNSRLCheckBox.toolTipText=Requires Hash Set service to had run previously, or be selected for next ingest. KeywordSearchGlobalSearchSettingsPanel.skipNSRLCheckBox.text=Do not add files in NSRL (known files) to keyword index during ingest KeywordSearchGlobalSearchSettingsPanel.informationLabel.text=Information @@ -219,7 +224,11 @@ KeywordSearchGlobalSearchSettingsPanel.filesIndexedValue.text=0 KeywordSearchGlobalSearchSettingsPanel.filesIndexedLabel.text=Files in keyword index: KeywordSearchGlobalSearchSettingsPanel.showSnippetsCB.text=Show Keyword Preview in Keyword Search Results (will result in longer search times) KeywordSearchGlobalSearchSettingsPanel.chunksValLabel.text=0 +KeywordSearchGlobalSearchSettingsPanel.timeRadioButton4.toolTipText=1 minute (overall ingest time will be longest) +KeywordSearchGlobalSearchSettingsPanel.timeRadioButton4.text_1=1 minute (faster feedback, longest ingest) KeywordSearchGlobalSearchSettingsPanel.chunksLabel.text=Chunks in keyword index: +KeywordSearchGlobalSearchSettingsPanel.timeRadioButton3.toolTipText=5 minutes (overall ingest time will be longer) +KeywordSearchGlobalSearchSettingsPanel.timeRadioButton3.text=5 minutes (default) KeywordSearchIngestModule.regExpHitLbl=Reg Ex hit: KeywordSearchIngestModule.kwHitLbl=Keyword hit: KeywordSearchIngestModule.kwHitThLbl=Keyword @@ -245,6 +254,8 @@ KeywordSearchListsManagementPanel.newKeywordListDescription2=Keyword list <{0}> KeywordSearchModuleFactory.getIngestJobSettingsPanel.exception.msg=Expected settings argument to be instanceof KeywordSearchJobSettings KeywordSearchModuleFactory.createFileIngestModule.exception.msg=Expected settings argument to be instanceof KeywordSearchJobSettings SearchRunner.Searcher.done.err.msg=Error performing keyword search +KeywordSearchGlobalSearchSettingsPanel.timeRadioButton5.toolTipText=Fastest overall, but no results until the end +KeywordSearchGlobalSearchSettingsPanel.timeRadioButton5.text=No periodic searches SolrConnectionCheck.HostnameOrPort=Invalid hostname and/or port number. SolrConnectionCheck.Hostname=Invalid hostname. SolrConnectionCheck.MissingHostname=Missing hostname. diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED index 8995ffc4951af4181984724431e306c3ad3e9ee0..44629aac36f7637717fd33a7ff4388ab2b60ddcf 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle.properties-MERGED @@ -18,8 +18,9 @@ ExtractAllTermsReport.getName.text=Extract Unique Words # {0} - Number of extracted terms ExtractAllTermsReport.numberExtractedTerms=Extracted {0} terms... ExtractAllTermsReport.search.ingestInProgressBody=<html>Keyword Search Ingest is currently running.<br />Not all files have been indexed and unique word extraction might yield incomplete results.<br />Do you want to proceed with unique word extraction anyway?</html> -ExtractAllTermsReport.search.noFilesInIdxMsg=No files are in index yet. If Solr keyword search indexing and Solr indexing were enabled, wait for ingest to complete. -ExtractAllTermsReport.search.noFilesInIdxMsg2=No files are in index yet. Re-ingest the image with the Keyword Search Module and Solr indexing enabled. +# {0} - Keyword search commit frequency +ExtractAllTermsReport.search.noFilesInIdxMsg=No files are in index yet. Try again later. Index is updated every {0} minutes. +ExtractAllTermsReport.search.noFilesInIdxMsg2=No files are in index yet. Try again later ExtractAllTermsReport.search.searchIngestInProgressTitle=Keyword Search Ingest in Progress ExtractAllTermsReport.startExport=Starting Unique Word Extraction ExtractedContentPanel.setMarkup.panelTxt=<span style='font-style:italic'>Loading text... Please wait</span> @@ -90,8 +91,8 @@ AbstractKeywordSearchPerformer.search.invalidSyntaxHeader=Invalid query statemen AbstractKeywordSearchPerformer.search.searchIngestInProgressTitle=Keyword Search Ingest in Progress AbstractKeywordSearchPerformer.search.ingestInProgressBody=<html>Keyword Search Ingest is currently running.<br />Not all files have been indexed and this search might yield incomplete results.<br />Do you want to proceed with this search anyway?</html> AbstractKeywordSearchPerformer.search.emptyKeywordErrorBody=Keyword list is empty, please add at least one keyword to the list -AbstractKeywordSearchPerformer.search.noFilesInIdxMsg=<html>No files are in index yet. <br />If Solr keyword search indexing was enabled, wait for ingest to complete</html> -AbstractKeywordSearchPerformer.search.noFilesIdxdMsg=<html>No files were indexed.<br />Re-ingest the image with the Keyword Search Module and Solr indexing enabled. </html> +AbstractKeywordSearchPerformer.search.noFilesInIdxMsg=<html>No files are in index yet. <br />Try again later. Index is updated every {0} minutes.</html> +AbstractKeywordSearchPerformer.search.noFilesIdxdMsg=<html>No files were indexed.<br />Re-ingest the image with the Keyword Search Module enabled. </html> ExtractedContentViewer.toolTip=Displays extracted text from files and keyword-search results. Requires Keyword Search ingest to be run on a file to activate this viewer. ExtractedContentViewer.getTitle=Indexed Text HighlightedMatchesSource.toString=Search Results @@ -226,6 +227,7 @@ KeywordSearchSettings.properties_options.text={0}_Options KeywordSearchSettings.propertiesNSRL.text={0}_NSRL KeywordSearchSettings.propertiesScripts.text={0}_Scripts NoOpenCoreException.err.noOpenSorlCore.msg=No currently open Solr core. +SearchRunner.query.exception.msg=Error performing query: # {0} - colelction name Server.deleteCore.exception.msg=Failed to delete Solr colelction {0} Server.exceptionMessage.unableToBackupCollection=Unable to backup Solr collection @@ -270,6 +272,11 @@ KeywordSearchGlobalLanguageSettingsPanel.enableUTF8Checkbox.text=Enable UTF8 tex KeywordSearchGlobalLanguageSettingsPanel.ingestSettingsLabel.text=Ingest settings for string extraction from unknown file types (changes effective on next ingest): KeywordSearchGlobalLanguageSettingsPanel.enableUTF16Checkbox.text=Enable UTF16LE and UTF16BE string extraction KeywordSearchGlobalLanguageSettingsPanel.languagesLabel.text=Enabled scripts (languages): +KeywordSearchGlobalSearchSettingsPanel.timeRadioButton1.toolTipText=20 mins. (fastest ingest time) +KeywordSearchGlobalSearchSettingsPanel.timeRadioButton1.text=20 minutes (slowest feedback, fastest ingest) +KeywordSearchGlobalSearchSettingsPanel.timeRadioButton2.toolTipText=10 minutes (faster overall ingest time than default) +KeywordSearchGlobalSearchSettingsPanel.timeRadioButton2.text=10 minutes (slower feedback, faster ingest) +KeywordSearchGlobalSearchSettingsPanel.frequencyLabel.text=Results update frequency during ingest: KeywordSearchGlobalSearchSettingsPanel.skipNSRLCheckBox.toolTipText=Requires Hash Set service to had run previously, or be selected for next ingest. KeywordSearchGlobalSearchSettingsPanel.skipNSRLCheckBox.text=Do not add files in NSRL (known files) to keyword index during ingest KeywordSearchGlobalSearchSettingsPanel.informationLabel.text=Information @@ -278,7 +285,11 @@ KeywordSearchGlobalSearchSettingsPanel.filesIndexedValue.text=0 KeywordSearchGlobalSearchSettingsPanel.filesIndexedLabel.text=Files in keyword index: KeywordSearchGlobalSearchSettingsPanel.showSnippetsCB.text=Show Keyword Preview in Keyword Search Results (will result in longer search times) KeywordSearchGlobalSearchSettingsPanel.chunksValLabel.text=0 +KeywordSearchGlobalSearchSettingsPanel.timeRadioButton4.toolTipText=1 minute (overall ingest time will be longest) +KeywordSearchGlobalSearchSettingsPanel.timeRadioButton4.text_1=1 minute (faster feedback, longest ingest) KeywordSearchGlobalSearchSettingsPanel.chunksLabel.text=Chunks in keyword index: +KeywordSearchGlobalSearchSettingsPanel.timeRadioButton3.toolTipText=5 minutes (overall ingest time will be longer) +KeywordSearchGlobalSearchSettingsPanel.timeRadioButton3.text=5 minutes (default) KeywordSearchIngestModule.regExpHitLbl=Reg Ex hit: KeywordSearchIngestModule.kwHitLbl=Keyword hit: KeywordSearchIngestModule.kwHitThLbl=Keyword @@ -304,6 +315,8 @@ KeywordSearchListsManagementPanel.newKeywordListDescription2=Keyword list <{0}> KeywordSearchModuleFactory.getIngestJobSettingsPanel.exception.msg=Expected settings argument to be instanceof KeywordSearchJobSettings KeywordSearchModuleFactory.createFileIngestModule.exception.msg=Expected settings argument to be instanceof KeywordSearchJobSettings SearchRunner.Searcher.done.err.msg=Error performing keyword search +KeywordSearchGlobalSearchSettingsPanel.timeRadioButton5.toolTipText=Fastest overall, but no results until the end +KeywordSearchGlobalSearchSettingsPanel.timeRadioButton5.text=No periodic searches Server.status.failed.msg=Local Solr server did not respond to status request. This may be because the server failed to start or is taking too long to initialize. SolrConnectionCheck.HostnameOrPort=Invalid hostname and/or port number. SolrConnectionCheck.Hostname=Invalid hostname. @@ -358,6 +371,7 @@ SolrSearchService.exceptionMessage.noCurrentSolrCore=IndexMetadata did not conta SolrSearchService.exceptionMessage.noIndexMetadata=Unable to create IndexMetaData from case directory: {0} # {0} - collection name SolrSearchService.exceptionMessage.unableToDeleteCollection=Unable to delete collection {0} +SolrSearchService.indexingError=Unable to index blackboard artifact. SolrSearchService.ServiceName=Solr Keyword Search Service SolrSearchService.DeleteDataSource.msg=Error Deleting Solr data for data source id {0} DropdownSingleTermSearchPanel.dataSourceCheckBox.text=Restrict search to the selected data sources: diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle_ja.properties b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle_ja.properties index 83e614bc936adff3ae6fd8df262d55a816a5bb8a..46af934d7c8949d3c41fe8ea0cb7bede5421f775 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle_ja.properties +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/Bundle_ja.properties @@ -7,6 +7,8 @@ AbstractKeywordSearchPerformer.search.dialogErrorHeader=\u30ad\u30fc\u30ef\u30fc AbstractKeywordSearchPerformer.search.emptyKeywordErrorBody=\u30ad\u30fc\u30ef\u30fc\u30c9\u30ea\u30b9\u30c8\u304c\u7a7a(\u672a\u5165\u529b)\u3067\u3059\u3002\u5c11\u306a\u304f\u3068\u30821\u3064\u306e\u30ad\u30fc\u30ef\u30fc\u30c9\u3092\u30ea\u30b9\u30c8\u306b\u8ffd\u52a0\u3057\u3066\u304f\u3060\u3055\u3044\u3002 AbstractKeywordSearchPerformer.search.ingestInProgressBody=<html>\u30ad\u30fc\u30ef\u30fc\u30c9\u691c\u7d22\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u304c\u73fe\u5728\u5b9f\u884c\u4e2d\u3067\u3059\u3002<br />\u3059\u3079\u3066\u306e\u30d5\u30a1\u30a4\u30eb\u304c\u30a4\u30f3\u30c7\u30c3\u30af\u30b9\u3055\u308c\u306a\u304b\u3063\u305f\u305f\u3081\u3001\u3053\u306e\u691c\u7d22\u306f\u4e0d\u5b8c\u5168\u306a\u7d50\u679c\u3092\u751f\u6210\u3059\u308b\u53ef\u80fd\u6027\u304c\u3042\u308a\u307e\u3059\u3002<br />\u305d\u308c\u3067\u3082\u3053\u306e\u691c\u7d22\u3092\u7d9a\u884c\u3057\u307e\u3059\u304b?</html> AbstractKeywordSearchPerformer.search.invalidSyntaxHeader=\u7121\u52b9\u306a\u30af\u30a8\u30ea\u30fb\u30b9\u30c6\u30fc\u30c8\u30e1\u30f3\u30c8\u3002 \u5185\u5bb9\u304c\u6b63\u898f\u8868\u73fe\u306e\u5834\u5408\u3001Lucene\u6b63\u898f\u8868\u73fe\u30d1\u30bf\u30fc\u30f3\u306e\u307f\u304c\u30b5\u30dd\u30fc\u30c8\u3055\u308c\u307e\u3059\u3002 POSIX\u6587\u5b57\u30af\u30e9\u30b9\uff08\\ n\u3084\\ w\u306a\u3069\uff09\u306f\u7121\u52b9\u3067\u3059\u3002 +AbstractKeywordSearchPerformer.search.noFilesIdxdMsg=<html>\u30d5\u30a1\u30a4\u30eb\u304c\u30a4\u30f3\u30c7\u30c3\u30af\u30b9\u3055\u308c\u3066\u3044\u307e\u305b\u3093\u3002<br />\u30ad\u30fc\u30ef\u30fc\u30c9\u691c\u7d22\u30e2\u30b8\u30e5\u30fc\u30eb\u3092\u6709\u52b9\u306b\u3057\u305f\u72b6\u614b\u3067\u30a4\u30e1\u30fc\u30b8\u3092\u518d\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u3057\u3066\u304f\u3060\u3055\u3044\u3002</html> +AbstractKeywordSearchPerformer.search.noFilesInIdxMsg=<html>\u307e\u3060\u30d5\u30a1\u30a4\u30eb\u304c\u7d22\u5f15\u306b\u542b\u307e\u308c\u3066\u3044\u307e\u305b\u3093\u3002<br />\u5f8c\u3067\u3082\u3046\u4e00\u5ea6\u304a\u8a66\u3057\u304f\u3060\u3055\u3044\u3002 \u7d22\u5f15\u306f {0} \u5206\u3054\u3068\u306b\u66f4\u65b0\u3055\u308c\u307e\u3059\u3002</html> AbstractKeywordSearchPerformer.search.searchIngestInProgressTitle=\u30ad\u30fc\u30ef\u30fc\u30c9\u691c\u7d22\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u304c\u9032\u884c\u4e2d\u3067\u3059 AccountsText.creditCardNumber=\u30af\u30ec\u30b8\u30c3\u30c8\u30ab\u30fc\u30c9\u756a\u53f7 AccountsText.creditCardNumbers=\u30af\u30ec\u30b8\u30c3\u30c8\u30ab\u30fc\u30c9\u756a\u53f7 @@ -53,6 +55,8 @@ ExtractAllTermsReport.exportComplete=\u30e6\u30cb\u30fc\u30af\u306a\u5358\u8a9e\ ExtractAllTermsReport.getName.text=\u30e6\u30cb\u30fc\u30af\u306a\u5358\u8a9e\u3092\u62bd\u51fa\u3059\u308b ExtractAllTermsReport.numberExtractedTerms=\u62bd\u51fa\u3055\u308c\u305f{0}\u7528\u8a9e... ExtractAllTermsReport.search.ingestInProgressBody=<html> \u30ad\u30fc\u30ef\u30fc\u30c9\u691c\u7d22\u8aad\u8fbc\u306f\u73fe\u5728\u5b9f\u884c\u4e2d\u3067\u3059\u3002<br/>\u3059\u3079\u3066\u306e\u30d5\u30a1\u30a4\u30eb\u304c\u30a4\u30f3\u30c7\u30c3\u30af\u30b9\u306b\u767b\u9332\u3055\u308c\u3066\u3044\u308b\u308f\u3051\u3067\u306f\u306a\u304f\u3001\u30e6\u30cb\u30fc\u30af\u306a\u5358\u8a9e\u3092\u62bd\u51fa\u306f\u4e0d\u5b8c\u5168\u306a\u7d50\u679c\u306b\u306a\u308b\u53ef\u80fd\u6027\u304c\u3042\u308a\u307e\u3059\u3002<br />\u305d\u308c\u3067\u3082\u30e6\u30cb\u30fc\u30af\u306a\u5358\u8a9e\u306e\u62bd\u51fa\u3092\u7d9a\u884c\u3057\u307e\u3059\u304b\uff1f</ html> +ExtractAllTermsReport.search.noFilesInIdxMsg=\u307e\u3060\u30a4\u30f3\u30c7\u30c3\u30af\u30b9\u306b\u767b\u9332\u3055\u308c\u3066\u3044\u308b\u30d5\u30a1\u30a4\u30eb\u306f\u3042\u308a\u307e\u305b\u3093\u3002 \u3042\u3068\u3067\u3082\u3046\u4e00\u5ea6\u8a66\u3057\u3066\u307f\u3066\u304f\u3060\u3055\u3044\u3002 \u30a4\u30f3\u30c7\u30c3\u30af\u30b9\u306f{0}\u5206\u3054\u3068\u306b\u66f4\u65b0\u3055\u308c\u307e\u3059\u3002 +ExtractAllTermsReport.search.noFilesInIdxMsg2=\u307e\u3060\u30a4\u30f3\u30c7\u30c3\u30af\u30b9\u306b\u767b\u9332\u3055\u308c\u3066\u3044\u308b\u30d5\u30a1\u30a4\u30eb\u306f\u3042\u308a\u307e\u305b\u3093\u3002 \u3042\u3068\u3067\u3082\u3046\u4e00\u5ea6\u8a66\u3057\u3066\u307f\u3066\u304f\u3060\u3055\u3044\u3002 ExtractAllTermsReport.search.searchIngestInProgressTitle=\u30ad\u30fc\u30ef\u30fc\u30c9\u691c\u7d22\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u304c\u9032\u884c\u4e2d\u3067\u3059 ExtractAllTermsReport.startExport=\u30e6\u30cb\u30fc\u30af\u306a\u5358\u8a9e\u62bd\u51fa\u306e\u958b\u59cb ExtractedContentPanel.SetMarkup.progress.loading={0} \u306e\u30c6\u30ad\u30b9\u30c8\u3092\u8aad\u307f\u8fbc\u3093\u3067\u3044\u307e\u3059 @@ -206,12 +210,23 @@ KeywordSearchGlobalSearchSettingsPanel.customizeComponents.windowsLimitedOCR=\u3 KeywordSearchGlobalSearchSettingsPanel.customizeComponents.windowsOCR=OCR\u6587\u5b57\u8a8d\u8b58\u3092\u6709\u52b9\u306b\u3059\u308b\uff08Windows 64\u30d3\u30c3\u30c8\u304c\u5fc5\u8981\uff09 KeywordSearchGlobalSearchSettingsPanel.filesIndexedLabel.text=\u30ad\u30fc\u30ef\u30fc\u30c9\u7d22\u5f15\u5185\u306e\u30d5\u30a1\u30a4\u30eb\: KeywordSearchGlobalSearchSettingsPanel.filesIndexedValue.text=0 +KeywordSearchGlobalSearchSettingsPanel.frequencyLabel.text=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u4e2d\u306e\u7d50\u679c\u66f4\u65b0\u983b\u5ea6\: KeywordSearchGlobalSearchSettingsPanel.informationLabel.text=\u60c5\u5831 KeywordSearchGlobalSearchSettingsPanel.ingestWarningLabel.text=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u304c\u9032\u884c\u4e2d\u3067\u3059\u3002\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u304c\u5b8c\u4e86\u3059\u308b\u307e\u3067\u4e00\u90e8\u306e\u8a2d\u5b9a\u3092\u5229\u7528\u3067\u304d\u307e\u305b\u3093\u3002 KeywordSearchGlobalSearchSettingsPanel.settingsLabel.text=\u8a2d\u5b9a KeywordSearchGlobalSearchSettingsPanel.showSnippetsCB.text=\u30ad\u30fc\u30ef\u30fc\u30c9\u691c\u7d22\u7d50\u679c\u306b\u30ad\u30fc\u30ef\u30fc\u30c9\u30d7\u30ec\u30d3\u30e5\u30fc\u3092\u8868\u793a(\u691c\u7d22\u6642\u9593\u304c\u9577\u304f\u306a\u308a\u307e\u3059) KeywordSearchGlobalSearchSettingsPanel.skipNSRLCheckBox.text=\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u4e2d\u306bNSRL(\u65e2\u77e5\u306e\u30d5\u30a1\u30a4\u30eb)\u306e\u30d5\u30a1\u30a4\u30eb\u3092\u30ad\u30fc\u30ef\u30fc\u30c9\u306b\u8ffd\u52a0\u3057\u306a\u3044\u3067\u304f\u3060\u3055\u3044 KeywordSearchGlobalSearchSettingsPanel.skipNSRLCheckBox.toolTipText=\u30cf\u30c3\u30b7\u30e5\u30bb\u30c3\u30c8\u30b5\u30fc\u30d3\u30b9\u306b\u4ee5\u524d\u306b\u5b9f\u884c\u6e08\u307f\u3067\u3042\u308b\u3053\u3068\u3001\u307e\u305f\u306f\u6b21\u306e\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u306b\u9078\u629e\u3055\u308c\u308b\u3053\u3068\u3092\u8981\u6c42\u3057\u307e\u3059\u3002 +KeywordSearchGlobalSearchSettingsPanel.timeRadioButton1.text=20\u5206(\u6700\u9045\u30d5\u30a3\u30fc\u30c9\u30d0\u30c3\u30af\u3001\u6700\u901f\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8) +KeywordSearchGlobalSearchSettingsPanel.timeRadioButton1.toolTipText=20\u5206(\u6700\u901f\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u6642\u9593) +KeywordSearchGlobalSearchSettingsPanel.timeRadioButton2.text=10\u5206(\u3088\u308a\u9045\u3044\u30d5\u30a3\u30fc\u30c9\u30d0\u30c3\u30af\u3001\u3088\u308a\u901f\u3044\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8) +KeywordSearchGlobalSearchSettingsPanel.timeRadioButton2.toolTipText=10\u5206(\u30c7\u30d5\u30a9\u30eb\u30c8\u3088\u308a\u3082\u901f\u3044\u7dcf\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u6642\u9593) +KeywordSearchGlobalSearchSettingsPanel.timeRadioButton3.text=5\u5206(\u30c7\u30d5\u30a9\u30eb\u30c8) +KeywordSearchGlobalSearchSettingsPanel.timeRadioButton3.toolTipText=5\u5206(\u7dcf\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u6642\u9593\u304c\u9577\u304f\u306a\u308a\u307e\u3059) +KeywordSearchGlobalSearchSettingsPanel.timeRadioButton4.text_1=1\u5206(\u3088\u308a\u901f\u3044\u30d5\u30a3\u30fc\u30c9\u30d0\u30c3\u30af\u3001\u6700\u9577\u306e\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8) +KeywordSearchGlobalSearchSettingsPanel.timeRadioButton4.toolTipText=1\u5206(\u7dcf\u30a4\u30f3\u30b8\u30a7\u30b9\u30c8\u6642\u9593\u306f\u6700\u9577\u306b\u306a\u308a\u307e\u3059) +KeywordSearchGlobalSearchSettingsPanel.timeRadioButton5.text=\u5b9a\u671f\u691c\u7d22\u306a\u3057 +KeywordSearchGlobalSearchSettingsPanel.timeRadioButton5.toolTipText=\u5168\u4f53\u3067\u6700\u901f\u3067\u3059\u304c\u3001\u6700\u5f8c\u307e\u3067\u7d50\u679c\u306f\u8868\u793a\u3055\u308c\u307e\u305b\u3093 KeywordSearchGlobalSettingsPanel.Title=\u30ad\u30fc\u30ef\u30fc\u30c9\u4e00\u62ec\u691c\u7d22\u8a2d\u5b9a KeywordSearchIngestModule.doInBackGround.displayName=\u30ad\u30fc\u30ef\u30fc\u30c9\u5b9a\u671f\u691c\u7d22 KeywordSearchIngestModule.doInBackGround.finalizeMsg=\u78ba\u5b9a diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownListSearchPanel.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownListSearchPanel.java index f53dc547c56f8ad13cf885fdee9f902dada70efc..7b84080217522196ec89e3e8f6e7a2426d717e4c 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownListSearchPanel.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/DropdownListSearchPanel.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011-2022 Basis Technology Corp. + * Copyright 2011-2018 Basis Technology Corp. * Contact: carrier <at> sleuthkit <dot> org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -30,6 +30,7 @@ import java.util.Iterator; import java.util.List; import java.util.Set; +import java.util.logging.Level; import javax.swing.JCheckBox; import javax.swing.JTable; import javax.swing.ListSelectionModel; @@ -142,7 +143,10 @@ public void propertyChange(PropertyChangeEvent evt) { searchAddListener = new ActionListener() { @Override public void actionPerformed(ActionEvent e) { - if (!ingestRunning) { + if (ingestRunning) { + IngestSearchRunner.getInstance().addKeywordListsToAllJobs(listsTableModel.getSelectedLists()); + logger.log(Level.INFO, "Submitted enqueued lists to ingest"); //NON-NLS + } else { searchAction(e); } } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractAllTermsReport.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractAllTermsReport.java index 584757aa93889efc7a5f7c46358036cd21d0955b..88178b42ea88f20c49730a3b942cb4bd353b936d 100755 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractAllTermsReport.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/ExtractAllTermsReport.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2022 Basis Technology Corp. + * Copyright 2021 Basis Technology Corp. * Contact: carrier <at> sleuthkit <dot> org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -51,8 +51,9 @@ public String getName() { @NbBundle.Messages({ "ExtractAllTermsReport.error.noOpenCase=No currently open case.", - "ExtractAllTermsReport.search.noFilesInIdxMsg=No files are in index yet. If Solr keyword search indexing and Solr indexing were enabled, wait for ingest to complete.", - "ExtractAllTermsReport.search.noFilesInIdxMsg2=No files are in index yet. Re-ingest the image with the Keyword Search Module and Solr indexing enabled.", + "# {0} - Keyword search commit frequency", + "ExtractAllTermsReport.search.noFilesInIdxMsg=No files are in index yet. Try again later. Index is updated every {0} minutes.", + "ExtractAllTermsReport.search.noFilesInIdxMsg2=No files are in index yet. Try again later", "ExtractAllTermsReport.search.searchIngestInProgressTitle=Keyword Search Ingest in Progress", "ExtractAllTermsReport.search.ingestInProgressBody=<html>Keyword Search Ingest is currently running.<br />Not all files have been indexed and unique word extraction might yield incomplete results.<br />Do you want to proceed with unique word extraction anyway?</html>", "ExtractAllTermsReport.startExport=Starting Unique Word Extraction", @@ -82,7 +83,7 @@ public void generateReport(GeneralReportSettings settings, ReportProgressPanel p if (filesIndexed == 0) { if (isIngestRunning) { - progressPanel.complete(ReportProgressPanel.ReportStatus.ERROR, Bundle.ExtractAllTermsReport_search_noFilesInIdxMsg()); + progressPanel.complete(ReportProgressPanel.ReportStatus.ERROR, Bundle.ExtractAllTermsReport_search_noFilesInIdxMsg(KeywordSearchSettings.getUpdateFrequency().getTime())); } else { progressPanel.complete(ReportProgressPanel.ReportStatus.ERROR, Bundle.ExtractAllTermsReport_search_noFilesInIdxMsg2()); } diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IngestSearchRunner.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IngestSearchRunner.java new file mode 100755 index 0000000000000000000000000000000000000000..9cd33a81674d286b2a3e64c3496734706019421e --- /dev/null +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/IngestSearchRunner.java @@ -0,0 +1,705 @@ +/* + * Autopsy Forensic Browser + * + * Copyright 2014 - 2021 Basis Technology Corp. + * Contact: carrier <at> sleuthkit <dot> org + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.sleuthkit.autopsy.keywordsearch; + +import com.google.common.util.concurrent.ThreadFactoryBuilder; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.concurrent.CancellationException; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.Future; +import java.util.concurrent.ScheduledThreadPoolExecutor; +import static java.util.concurrent.TimeUnit.MILLISECONDS; +import java.util.concurrent.atomic.AtomicLong; +import java.util.logging.Level; +import javax.annotation.concurrent.GuardedBy; +import javax.swing.SwingUtilities; +import javax.swing.SwingWorker; +import org.netbeans.api.progress.ProgressHandle; +import org.openide.util.Cancellable; +import org.openide.util.NbBundle; +import org.openide.util.NbBundle.Messages; +import org.sleuthkit.autopsy.core.RuntimeProperties; +import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.autopsy.coreutils.MessageNotifyUtil; +import org.sleuthkit.autopsy.coreutils.StopWatch; +import org.sleuthkit.autopsy.coreutils.ThreadConfined; +import org.sleuthkit.autopsy.ingest.IngestJobContext; +import org.sleuthkit.autopsy.ingest.IngestMessage; +import org.sleuthkit.autopsy.ingest.IngestServices; + +/** + * Performs periodic and final keyword searches for ingest jobs. Periodic + * searches are done in background tasks. This represents a careful working + * around of the contract for IngestModule.process(). Final searches are done + * synchronously in the calling thread, as required by the contract for + * IngestModule.shutDown(). + */ +final class IngestSearchRunner { + + private static final Logger logger = Logger.getLogger(IngestSearchRunner.class.getName()); + private static IngestSearchRunner instance = null; + private final IngestServices services = IngestServices.getInstance(); + private Ingester ingester = null; + private long currentUpdateIntervalMs; + private volatile boolean periodicSearchTaskRunning; + private volatile Future<?> periodicSearchTaskHandle; + private final ScheduledThreadPoolExecutor periodicSearchTaskExecutor; + private static final int NUM_SEARCH_SCHEDULING_THREADS = 1; + private static final String SEARCH_SCHEDULER_THREAD_NAME = "periodic-search-scheduling-%d"; + private final Map<Long, SearchJobInfo> jobs = new ConcurrentHashMap<>(); // Ingest job ID to search job info + private final boolean usingNetBeansGUI = RuntimeProperties.runningWithGUI(); + + /* + * Constructs a singleton object that performs periodic and final keyword + * searches for ingest jobs. Periodic searches are done in background tasks. + * This represents a careful working around of the contract for + * IngestModule.process(). Final searches are done synchronously in the + * calling thread, as required by the contract for IngestModule.shutDown(). + */ + private IngestSearchRunner() { + currentUpdateIntervalMs = ((long) KeywordSearchSettings.getUpdateFrequency().getTime()) * 60 * 1000; + ingester = Ingester.getDefault(); + periodicSearchTaskExecutor = new ScheduledThreadPoolExecutor(NUM_SEARCH_SCHEDULING_THREADS, new ThreadFactoryBuilder().setNameFormat(SEARCH_SCHEDULER_THREAD_NAME).build()); + } + + /** + * Gets the ingest search runner singleton. + * + * @return The ingest search runner. + */ + public static synchronized IngestSearchRunner getInstance() { + if (instance == null) { + instance = new IngestSearchRunner(); + } + return instance; + } + + /** + * Starts the search job for an ingest job. + * + * @param jobContext The ingest job context. + * @param keywordListNames The names of the keyword search lists for the + * ingest job. + */ + public synchronized void startJob(IngestJobContext jobContext, List<String> keywordListNames) { + long jobId = jobContext.getJobId(); + if (jobs.containsKey(jobId) == false) { + SearchJobInfo jobData = new SearchJobInfo(jobContext, keywordListNames); + jobs.put(jobId, jobData); + } + + /* + * Keep track of the number of keyword search file ingest modules that + * are doing analysis for the ingest job, i.e., that have called this + * method. This is needed by endJob(). + */ + jobs.get(jobId).incrementModuleReferenceCount(); + + /* + * Start a periodic search task in the + */ + if ((jobs.size() > 0) && (periodicSearchTaskRunning == false)) { + currentUpdateIntervalMs = ((long) KeywordSearchSettings.getUpdateFrequency().getTime()) * 60 * 1000; + periodicSearchTaskHandle = periodicSearchTaskExecutor.schedule(new PeriodicSearchTask(), currentUpdateIntervalMs, MILLISECONDS); + periodicSearchTaskRunning = true; + } + } + + /** + * Finishes a search job for an ingest job. + * + * @param jobId The ingest job ID. + */ + public synchronized void endJob(long jobId) { + /* + * Only complete the job if this is the last keyword search file ingest + * module doing annalysis for this job. + */ + SearchJobInfo job; + job = jobs.get(jobId); + if (job == null) { + return; // RJCTODO: SEVERE + } + if (job.decrementModuleReferenceCount() != 0) { + jobs.remove(jobId); + } + + /* + * Commit the index and do the final search. The final search is done in + * the ingest thread that shutDown() on the keyword search file ingest + * module, per the contract of IngestModule.shutDwon(). + */ + logger.log(Level.INFO, "Commiting search index before final search for search job {0}", job.getJobId()); //NON-NLS + commit(); + logger.log(Level.INFO, "Starting final search for search job {0}", job.getJobId()); //NON-NLS + doFinalSearch(job); + logger.log(Level.INFO, "Final search for search job {0} completed", job.getJobId()); //NON-NLS + + if (jobs.isEmpty()) { + cancelPeriodicSearchSchedulingTask(); + } + } + + /** + * Stops the search job for an ingest job. + * + * @param jobId The ingest job ID. + */ + public synchronized void stopJob(long jobId) { + logger.log(Level.INFO, "Stopping search job {0}", jobId); //NON-NLS + commit(); + + SearchJobInfo job; + job = jobs.get(jobId); + if (job == null) { + return; + } + + /* + * Request cancellation of the current keyword search, whether it is a + * preiodic search or a final search. + */ + IngestSearchRunner.Searcher currentSearcher = job.getCurrentSearcher(); + if ((currentSearcher != null) && (!currentSearcher.isDone())) { + logger.log(Level.INFO, "Cancelling search job {0}", jobId); //NON-NLS + currentSearcher.cancel(true); + } + + jobs.remove(jobId); + + if (jobs.isEmpty()) { + cancelPeriodicSearchSchedulingTask(); + } + } + + /** + * Adds the given keyword list names to the set of keyword lists to be + * searched by ALL keyword search jobs. This supports adding one or more + * keyword search lists to ingest jobs already in progress. + * + * @param keywordListNames The n ames of the additional keyword lists. + */ + public synchronized void addKeywordListsToAllJobs(List<String> keywordListNames) { + for (String listName : keywordListNames) { + logger.log(Level.INFO, "Adding keyword list {0} to all jobs", listName); //NON-NLS + for (SearchJobInfo j : jobs.values()) { + j.addKeywordListName(listName); + } + } + } + + /** + * Commits the Solr index for the current case and publishes an event + * indicating the current number of indexed items (this is no longer just + * files). + */ + private void commit() { + ingester.commit(); + + /* + * Publish an event advertising the number of indexed items. Note that + * this is no longer the number of indexed files, since the text of many + * items in addition to files is indexed. + */ + try { + final int numIndexedFiles = KeywordSearch.getServer().queryNumIndexedFiles(); + KeywordSearch.fireNumIndexedFilesChange(null, numIndexedFiles); + } catch (NoOpenCoreException | KeywordSearchModuleException ex) { + logger.log(Level.SEVERE, "Error executing Solr query for number of indexed files", ex); //NON-NLS + } + } + + /** + * Performs the final keyword search for an ingest job. The search is done + * synchronously, as required by the contract for IngestModule.shutDown(). + * + * @param job The keyword search job info. + */ + private void doFinalSearch(SearchJobInfo job) { + if (!job.getKeywordListNames().isEmpty()) { + try { + /* + * Wait for any periodic searches being done in a SwingWorker + * pool thread to finish. + */ + job.waitForCurrentWorker(); + IngestSearchRunner.Searcher finalSearcher = new IngestSearchRunner.Searcher(job, true); + job.setCurrentSearcher(finalSearcher); + /* + * Do the final search synchronously on the current ingest + * thread, per the contract specified + */ + finalSearcher.doInBackground(); + } catch (InterruptedException | CancellationException ex) { + logger.log(Level.INFO, "Final search for search job {0} interrupted or cancelled", job.getJobId()); //NON-NLS + } catch (Exception ex) { + logger.log(Level.SEVERE, String.format("Final search for search job %d failed", job.getJobId()), ex); //NON-NLS + } + } + } + + /** + * Cancels the current periodic search scheduling task. + */ + private synchronized void cancelPeriodicSearchSchedulingTask() { + if (periodicSearchTaskHandle != null) { + logger.log(Level.INFO, "No more search jobs, stopping periodic search scheduling"); //NON-NLS + periodicSearchTaskHandle.cancel(true); + periodicSearchTaskRunning = false; + } + } + + /** + * Task that runs in ScheduledThreadPoolExecutor to periodically start and + * wait for keyword search tasks for each keyword search job in progress. + * The keyword search tasks for individual ingest jobs are implemented as + * SwingWorkers to support legacy APIs. + */ + private final class PeriodicSearchTask implements Runnable { + + @Override + public void run() { + /* + * If there are no more jobs or this task has been cancelled, exit. + */ + if (jobs.isEmpty() || periodicSearchTaskHandle.isCancelled()) { + logger.log(Level.INFO, "Periodic search scheduling task has been cancelled, exiting"); //NON-NLS + periodicSearchTaskRunning = false; + return; + } + + /* + * Commit the Solr index for the current case before doing the + * searches. + */ + commit(); + + /* + * Do a keyword search for each ingest job in progress. When the + * searches are done, recalculate the "hold off" time between + * searches to prevent back-to-back periodic searches and schedule + * the nect periodic search task. + */ + final StopWatch stopWatch = new StopWatch(); + stopWatch.start(); + for (Iterator<Entry<Long, SearchJobInfo>> iterator = jobs.entrySet().iterator(); iterator.hasNext();) { + SearchJobInfo job = iterator.next().getValue(); + + if (periodicSearchTaskHandle.isCancelled()) { + logger.log(Level.INFO, "Periodic search scheduling task has been cancelled, exiting"); //NON-NLS + periodicSearchTaskRunning = false; + return; + } + + if (!job.getKeywordListNames().isEmpty() && !job.isWorkerRunning()) { + logger.log(Level.INFO, "Starting periodic search for search job {0}", job.getJobId()); + Searcher searcher = new Searcher(job, false); + job.setCurrentSearcher(searcher); + searcher.execute(); + job.setWorkerRunning(true); + try { + searcher.get(); + } catch (InterruptedException | ExecutionException ex) { + logger.log(Level.SEVERE, String.format("Error performing keyword search for ingest job %d", job.getJobId()), ex); //NON-NLS + services.postMessage(IngestMessage.createErrorMessage( + KeywordSearchModuleFactory.getModuleName(), + NbBundle.getMessage(this.getClass(), "SearchRunner.Searcher.done.err.msg"), ex.getMessage())); + } catch (java.util.concurrent.CancellationException ex) { + logger.log(Level.SEVERE, String.format("Keyword search for ingest job %d cancelled", job.getJobId()), ex); //NON-NLS + } + } + } + stopWatch.stop(); + logger.log(Level.INFO, "Periodic searches for all ingest jobs cumulatively took {0} secs", stopWatch.getElapsedTimeSecs()); //NON-NLS + recalculateUpdateIntervalTime(stopWatch.getElapsedTimeSecs()); // ELDEBUG + periodicSearchTaskHandle = periodicSearchTaskExecutor.schedule(new PeriodicSearchTask(), currentUpdateIntervalMs, MILLISECONDS); + } + + /** + * Sets the time interval between periodic keyword searches to avoid + * running back-to-back searches. If the most recent round of searches + * took longer that 1/4 of the current interval, doubles the interval. + * + * @param lastSerchTimeSec The time in seconds used to execute the most + * recent round of keword searches. + */ + private void recalculateUpdateIntervalTime(long lastSerchTimeSec) { + if (lastSerchTimeSec * 1000 < currentUpdateIntervalMs / 4) { + return; + } + currentUpdateIntervalMs *= 2; + logger.log(Level.WARNING, "Last periodic search took {0} sec. Increasing search interval to {1} sec", new Object[]{lastSerchTimeSec, currentUpdateIntervalMs / 1000}); + } + } + + /** + * A data structure to keep track of the keyword lists, current results, and + * search running status for an ingest job. + */ + private class SearchJobInfo { + + private final IngestJobContext jobContext; + private final long jobId; + private final long dataSourceId; + private volatile boolean workerRunning; + @GuardedBy("this") + private final List<String> keywordListNames; + @GuardedBy("this") + private final Map<Keyword, Set<Long>> currentResults; // Keyword to object IDs of items with hits + private IngestSearchRunner.Searcher currentSearcher; + private final AtomicLong moduleReferenceCount = new AtomicLong(0); + private final Object finalSearchLock = new Object(); + + private SearchJobInfo(IngestJobContext jobContext, List<String> keywordListNames) { + this.jobContext = jobContext; + jobId = jobContext.getJobId(); + dataSourceId = jobContext.getDataSource().getId(); + this.keywordListNames = new ArrayList<>(keywordListNames); + currentResults = new HashMap<>(); + workerRunning = false; + currentSearcher = null; + } + + private IngestJobContext getJobContext() { + return jobContext; + } + + private long getJobId() { + return jobId; + } + + private long getDataSourceId() { + return dataSourceId; + } + + private synchronized List<String> getKeywordListNames() { + return new ArrayList<>(keywordListNames); + } + + private synchronized void addKeywordListName(String keywordListName) { + if (!keywordListNames.contains(keywordListName)) { + keywordListNames.add(keywordListName); + } + } + + private synchronized Set<Long> currentKeywordResults(Keyword k) { + return currentResults.get(k); + } + + private synchronized void addKeywordResults(Keyword k, Set<Long> resultsIDs) { + currentResults.put(k, resultsIDs); + } + + private boolean isWorkerRunning() { + return workerRunning; + } + + private void setWorkerRunning(boolean flag) { + workerRunning = flag; + } + + private synchronized IngestSearchRunner.Searcher getCurrentSearcher() { + return currentSearcher; + } + + private synchronized void setCurrentSearcher(IngestSearchRunner.Searcher searchRunner) { + currentSearcher = searchRunner; + } + + private void incrementModuleReferenceCount() { + moduleReferenceCount.incrementAndGet(); + } + + private long decrementModuleReferenceCount() { + return moduleReferenceCount.decrementAndGet(); + } + + /** + * Waits for the current search task to complete. + * + * @throws InterruptedException + */ + private void waitForCurrentWorker() throws InterruptedException { + synchronized (finalSearchLock) { + while (workerRunning) { + logger.log(Level.INFO, String.format("Waiting for previous search task for job %d to finish", jobId)); //NON-NLS + finalSearchLock.wait(); + logger.log(Level.INFO, String.format("Notified previous search task for job %d to finish", jobId)); //NON-NLS + } + } + } + + /** + * Signals any threads waiting on the current search task to complete. + */ + private void searchNotify() { + synchronized (finalSearchLock) { + workerRunning = false; + finalSearchLock.notify(); + } + } + } + + /* + * A SwingWorker responsible for searching the Solr index of the current + * case for the keywords for an ingest job. Keyword hit analysis results are + * created and posted to the blackboard and notifications are sent to the + * ingest inbox. + */ + private final class Searcher extends SwingWorker<Object, Void> { + + /* + * Searcher has private copies/snapshots of the lists and keywords + */ + private final SearchJobInfo job; + private final List<Keyword> keywords; //keywords to search + private final List<String> keywordListNames; // lists currently being searched + private final List<KeywordList> keywordLists; + private final Map<Keyword, KeywordList> keywordToList; //keyword to list name mapping + @ThreadConfined(type = ThreadConfined.ThreadType.AWT) + private ProgressHandle progressIndicator; + private boolean finalRun = false; + + Searcher(SearchJobInfo job, boolean finalRun) { + this.job = job; + this.finalRun = finalRun; + keywordListNames = job.getKeywordListNames(); + keywords = new ArrayList<>(); + keywordToList = new HashMap<>(); + keywordLists = new ArrayList<>(); + } + + @Override + @Messages("SearchRunner.query.exception.msg=Error performing query:") + protected Object doInBackground() throws Exception { + try { + if (usingNetBeansGUI) { + /* + * If running in the NetBeans thick client application + * version of Autopsy, NetBeans progress handles (i.e., + * progress bars) are used to display search progress in the + * lower right hand corner of the main application window. + * + * A layer of abstraction to allow alternate representations + * of progress could be used here, as it is in other places + * in the application (see implementations and usage of + * org.sleuthkit.autopsy.progress.ProgressIndicator + * interface), to better decouple keyword search from the + * application's presentation layer. + */ + SwingUtilities.invokeAndWait(() -> { + final String displayName = NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.displayName") + + (finalRun ? (" - " + NbBundle.getMessage(this.getClass(), "KeywordSearchIngestModule.doInBackGround.finalizeMsg")) : ""); + progressIndicator = ProgressHandle.createHandle(displayName, new Cancellable() { + @Override + public boolean cancel() { + if (progressIndicator != null) { + progressIndicator.setDisplayName(displayName + " " + NbBundle.getMessage(this.getClass(), "SearchRunner.doInBackGround.cancelMsg")); + } + logger.log(Level.INFO, "Search cancelled by user"); //NON-NLS + new Thread(() -> { + IngestSearchRunner.Searcher.this.cancel(true); + }).start(); + return true; + } + }); + progressIndicator.start(); + progressIndicator.switchToIndeterminate(); + }); + } + + updateKeywords(); + for (Keyword keyword : keywords) { + if (isCancelled() || job.getJobContext().fileIngestIsCancelled()) { + logger.log(Level.INFO, "Cancellation requested, exiting before new keyword processed: {0}", keyword.getSearchTerm()); //NON-NLS + return null; + } + + KeywordList keywordList = keywordToList.get(keyword); + if (usingNetBeansGUI) { + String searchTermStr = keyword.getSearchTerm(); + if (searchTermStr.length() > 50) { + searchTermStr = searchTermStr.substring(0, 49) + "..."; + } + final String progressMessage = keywordList.getName() + ": " + searchTermStr; + SwingUtilities.invokeLater(() -> { + progressIndicator.progress(progressMessage); + }); + } + + // Filtering + //limit search to currently ingested data sources + //set up a filter with 1 or more image ids OR'ed + KeywordSearchQuery keywordSearchQuery = KeywordSearchUtil.getQueryForKeyword(keyword, keywordList); + KeywordQueryFilter dataSourceFilter = new KeywordQueryFilter(KeywordQueryFilter.FilterType.DATA_SOURCE, job.getDataSourceId()); + keywordSearchQuery.addFilter(dataSourceFilter); + + // Do the actual search + QueryResults queryResults; + try { + queryResults = keywordSearchQuery.performQuery(); + } catch (KeywordSearchModuleException | NoOpenCoreException ex) { + logger.log(Level.SEVERE, "Error performing query: " + keyword.getSearchTerm(), ex); //NON-NLS + if (usingNetBeansGUI) { + final String userMessage = Bundle.SearchRunner_query_exception_msg() + keyword.getSearchTerm(); + SwingUtilities.invokeLater(() -> { + MessageNotifyUtil.Notify.error(userMessage, ex.getCause().getMessage()); + }); + } + //no reason to continue with next query if recovery failed + //or wait for recovery to kick in and run again later + //likely case has closed and threads are being interrupted + return null; + } catch (CancellationException e) { + logger.log(Level.INFO, "Cancellation requested, exiting during keyword query: {0}", keyword.getSearchTerm()); //NON-NLS + return null; + } + + // Reduce the results of the query to only those hits we + // have not already seen. + QueryResults newResults = filterResults(queryResults); + + if (!newResults.getKeywords().isEmpty()) { + // Create blackboard artifacts + newResults.process(this, keywordList.getIngestMessages(), true, job.getJobId()); + } + } + } catch (Exception ex) { + logger.log(Level.SEVERE, String.format("Error performing keyword search for ingest job %d", job.getJobId()), ex); //NON-NLS + } finally { + if (progressIndicator != null) { + SwingUtilities.invokeLater(new Runnable() { + @Override + public void run() { + progressIndicator.finish(); + progressIndicator = null; + } + }); + } + // In case a thread is waiting on this worker to be done + job.searchNotify(); + } + + return null; + } + + /** + * Sync-up the updated keywords from the currently used lists in the XML + */ + private void updateKeywords() { + XmlKeywordSearchList loader = XmlKeywordSearchList.getCurrent(); + + keywords.clear(); + keywordToList.clear(); + keywordLists.clear(); + + for (String name : keywordListNames) { + KeywordList list = loader.getList(name); + keywordLists.add(list); + for (Keyword k : list.getKeywords()) { + keywords.add(k); + keywordToList.put(k, list); + } + } + } + + /** + * This method filters out all of the hits found in earlier periodic + * searches and returns only the results found by the most recent + * search. + * + * This method will only return hits for objects for which we haven't + * previously seen a hit for the keyword. + * + * @param queryResult The results returned by a keyword search. + * + * @return A unique set of hits found by the most recent search for + * objects that have not previously had a hit. The hits will be + * for the lowest numbered chunk associated with the object. + * + */ + private QueryResults filterResults(QueryResults queryResult) { + + // Create a new (empty) QueryResults object to hold the most recently + // found hits. + QueryResults newResults = new QueryResults(queryResult.getQuery()); + + // For each keyword represented in the results. + for (Keyword keyword : queryResult.getKeywords()) { + // These are all of the hits across all objects for the most recent search. + // This may well include duplicates of hits we've seen in earlier periodic searches. + List<KeywordHit> queryTermResults = queryResult.getResults(keyword); + + // Sort the hits for this keyword so that we are always + // guaranteed to return the hit for the lowest chunk. + Collections.sort(queryTermResults); + + // This will be used to build up the hits we haven't seen before + // for this keyword. + List<KeywordHit> newUniqueHits = new ArrayList<>(); + + // Get the set of object ids seen in the past by this searcher + // for the given keyword. + Set<Long> curTermResults = job.currentKeywordResults(keyword); + if (curTermResults == null) { + // We create a new empty set if we haven't seen results for + // this keyword before. + curTermResults = new HashSet<>(); + } + + // For each hit for this keyword. + for (KeywordHit hit : queryTermResults) { + if (curTermResults.contains(hit.getSolrObjectId())) { + // Skip the hit if we've already seen a hit for + // this keyword in the object. + continue; + } + + // We haven't seen the hit before so add it to list of new + // unique hits. + newUniqueHits.add(hit); + + // Add the object id to the results we've seen for this + // keyword. + curTermResults.add(hit.getSolrObjectId()); + } + + // Update the job with the list of objects for which we have + // seen hits for the current keyword. + job.addKeywordResults(keyword, curTermResults); + + // Add the new hits for the current keyword into the results + // to be returned. + newResults.addResult(keyword, newUniqueHits); + } + + return newResults; + } + } + +} diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalSearchSettingsPanel.form b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalSearchSettingsPanel.form index 0f8800efdd0af3e8641397d43222f3c7a9791361..4a0cadaefd4c67c9bdea4ac3ea756b9e56353ac6 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalSearchSettingsPanel.form +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalSearchSettingsPanel.form @@ -1,6 +1,10 @@ <?xml version="1.0" encoding="UTF-8" ?> <Form version="1.5" maxVersion="1.7" type="org.netbeans.modules.form.forminfo.JPanelFormInfo"> + <NonVisualComponents> + <Component class="javax.swing.ButtonGroup" name="timeGroup"> + </Component> + </NonVisualComponents> <AuxValues> <AuxValue name="FormSettings_autoResourcing" type="java.lang.Integer" value="1"/> <AuxValue name="FormSettings_autoSetComponentName" type="java.lang.Boolean" value="false"/> @@ -43,13 +47,23 @@ <EmptySpace type="separate" max="-2" attributes="0"/> <Component id="filesIndexedValue" min="-2" max="-2" attributes="0"/> </Group> + <Component id="frequencyLabel" alignment="0" min="-2" max="-2" attributes="0"/> <Group type="102" alignment="0" attributes="0"> <Component id="chunksLabel" linkSize="1" min="-2" max="-2" attributes="0"/> <EmptySpace type="separate" max="-2" attributes="0"/> <Component id="chunksValLabel" min="-2" max="-2" attributes="0"/> </Group> + <Group type="102" alignment="0" attributes="0"> + <EmptySpace min="16" pref="16" max="-2" attributes="0"/> + <Group type="103" groupAlignment="0" attributes="0"> + <Component id="timeRadioButton2" min="-2" max="-2" attributes="0"/> + <Component id="timeRadioButton1" min="-2" max="-2" attributes="0"/> + <Component id="timeRadioButton3" alignment="0" min="-2" max="-2" attributes="0"/> + <Component id="timeRadioButton4" alignment="0" min="-2" max="-2" attributes="0"/> + <Component id="timeRadioButton5" alignment="0" min="-2" max="-2" attributes="0"/> + </Group> + </Group> </Group> - <EmptySpace min="-2" pref="132" max="-2" attributes="0"/> </Group> </Group> <EmptySpace max="32767" attributes="0"/> @@ -76,7 +90,19 @@ <Component id="skipNSRLCheckBox" min="-2" max="-2" attributes="0"/> <EmptySpace max="-2" attributes="0"/> <Component id="showSnippetsCB" min="-2" max="-2" attributes="0"/> - <EmptySpace type="unrelated" max="-2" attributes="0"/> + <EmptySpace max="-2" attributes="0"/> + <Component id="frequencyLabel" min="-2" max="-2" attributes="0"/> + <EmptySpace max="-2" attributes="0"/> + <Component id="timeRadioButton1" min="-2" max="-2" attributes="0"/> + <EmptySpace max="-2" attributes="0"/> + <Component id="timeRadioButton2" min="-2" max="-2" attributes="0"/> + <EmptySpace max="-2" attributes="0"/> + <Component id="timeRadioButton3" min="-2" max="-2" attributes="0"/> + <EmptySpace max="-2" attributes="0"/> + <Component id="timeRadioButton4" min="-2" max="-2" attributes="0"/> + <EmptySpace max="-2" attributes="0"/> + <Component id="timeRadioButton5" min="-2" max="-2" attributes="0"/> + <EmptySpace max="-2" attributes="0"/> <Group type="103" groupAlignment="1" attributes="0"> <Component id="informationLabel" min="-2" max="-2" attributes="0"/> <Component id="informationSeparator" min="-2" pref="7" max="-2" attributes="0"/> @@ -93,7 +119,7 @@ </Group> <EmptySpace type="unrelated" max="-2" attributes="0"/> <Component id="ingestWarningLabel" min="-2" max="-2" attributes="0"/> - <EmptySpace pref="151" max="32767" attributes="0"/> + <EmptySpace max="32767" attributes="0"/> </Group> </Group> </DimensionLayout> @@ -158,6 +184,65 @@ </Component> <Component class="javax.swing.JSeparator" name="informationSeparator"> </Component> + <Component class="javax.swing.JLabel" name="frequencyLabel"> + <Properties> + <Property name="text" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor"> + <ResourceString bundle="org/sleuthkit/autopsy/keywordsearch/Bundle.properties" key="KeywordSearchGlobalSearchSettingsPanel.frequencyLabel.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, "{key}")"/> + </Property> + </Properties> + </Component> + <Component class="javax.swing.JRadioButton" name="timeRadioButton1"> + <Properties> + <Property name="text" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor"> + <ResourceString bundle="org/sleuthkit/autopsy/keywordsearch/Bundle.properties" key="KeywordSearchGlobalSearchSettingsPanel.timeRadioButton1.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, "{key}")"/> + </Property> + <Property name="toolTipText" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor"> + <ResourceString bundle="org/sleuthkit/autopsy/keywordsearch/Bundle.properties" key="KeywordSearchGlobalSearchSettingsPanel.timeRadioButton1.toolTipText" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, "{key}")"/> + </Property> + </Properties> + <Events> + <EventHandler event="actionPerformed" listener="java.awt.event.ActionListener" parameters="java.awt.event.ActionEvent" handler="timeRadioButton1ActionPerformed"/> + </Events> + </Component> + <Component class="javax.swing.JRadioButton" name="timeRadioButton2"> + <Properties> + <Property name="text" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor"> + <ResourceString bundle="org/sleuthkit/autopsy/keywordsearch/Bundle.properties" key="KeywordSearchGlobalSearchSettingsPanel.timeRadioButton2.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, "{key}")"/> + </Property> + <Property name="toolTipText" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor"> + <ResourceString bundle="org/sleuthkit/autopsy/keywordsearch/Bundle.properties" key="KeywordSearchGlobalSearchSettingsPanel.timeRadioButton2.toolTipText" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, "{key}")"/> + </Property> + </Properties> + <Events> + <EventHandler event="actionPerformed" listener="java.awt.event.ActionListener" parameters="java.awt.event.ActionEvent" handler="timeRadioButton2ActionPerformed"/> + </Events> + </Component> + <Component class="javax.swing.JRadioButton" name="timeRadioButton3"> + <Properties> + <Property name="text" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor"> + <ResourceString bundle="org/sleuthkit/autopsy/keywordsearch/Bundle.properties" key="KeywordSearchGlobalSearchSettingsPanel.timeRadioButton3.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, "{key}")"/> + </Property> + <Property name="toolTipText" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor"> + <ResourceString bundle="org/sleuthkit/autopsy/keywordsearch/Bundle.properties" key="KeywordSearchGlobalSearchSettingsPanel.timeRadioButton3.toolTipText" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, "{key}")"/> + </Property> + </Properties> + <Events> + <EventHandler event="actionPerformed" listener="java.awt.event.ActionListener" parameters="java.awt.event.ActionEvent" handler="timeRadioButton3ActionPerformed"/> + </Events> + </Component> + <Component class="javax.swing.JRadioButton" name="timeRadioButton4"> + <Properties> + <Property name="text" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor"> + <ResourceString bundle="org/sleuthkit/autopsy/keywordsearch/Bundle.properties" key="KeywordSearchGlobalSearchSettingsPanel.timeRadioButton4.text_1" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, "{key}")"/> + </Property> + <Property name="toolTipText" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor"> + <ResourceString bundle="org/sleuthkit/autopsy/keywordsearch/Bundle.properties" key="KeywordSearchGlobalSearchSettingsPanel.timeRadioButton4.toolTipText" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, "{key}")"/> + </Property> + </Properties> + <Events> + <EventHandler event="actionPerformed" listener="java.awt.event.ActionListener" parameters="java.awt.event.ActionEvent" handler="timeRadioButton4ActionPerformed"/> + </Events> + </Component> <Component class="javax.swing.JCheckBox" name="showSnippetsCB"> <Properties> <Property name="text" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor"> @@ -168,6 +253,19 @@ <EventHandler event="actionPerformed" listener="java.awt.event.ActionListener" parameters="java.awt.event.ActionEvent" handler="showSnippetsCBActionPerformed"/> </Events> </Component> + <Component class="javax.swing.JRadioButton" name="timeRadioButton5"> + <Properties> + <Property name="text" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor"> + <ResourceString bundle="org/sleuthkit/autopsy/keywordsearch/Bundle.properties" key="KeywordSearchGlobalSearchSettingsPanel.timeRadioButton5.text" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, "{key}")"/> + </Property> + <Property name="toolTipText" type="java.lang.String" editor="org.netbeans.modules.i18n.form.FormI18nStringEditor"> + <ResourceString bundle="org/sleuthkit/autopsy/keywordsearch/Bundle.properties" key="KeywordSearchGlobalSearchSettingsPanel.timeRadioButton5.toolTipText" replaceFormat="org.openide.util.NbBundle.getMessage({sourceFileName}.class, "{key}")"/> + </Property> + </Properties> + <Events> + <EventHandler event="actionPerformed" listener="java.awt.event.ActionListener" parameters="java.awt.event.ActionEvent" handler="timeRadioButton5ActionPerformed"/> + </Events> + </Component> <Component class="javax.swing.JLabel" name="ingestWarningLabel"> <Properties> <Property name="icon" type="javax.swing.Icon" editor="org.netbeans.modules.form.editors2.IconEditor"> diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalSearchSettingsPanel.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalSearchSettingsPanel.java index 7393e3ea4dec9d720d69b6f0e53261aa1ba29bbc..ccd1de71dac3c0408c23a03e821589de28022d60 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalSearchSettingsPanel.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchGlobalSearchSettingsPanel.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2012-2022 Basis Technology Corp. + * Copyright 2012-2018 Basis Technology Corp. * Contact: carrier <at> sleuthkit <dot> org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -26,7 +26,9 @@ import org.openide.util.NbBundle; import org.sleuthkit.autopsy.corecomponents.OptionsPanel; import org.sleuthkit.autopsy.coreutils.Logger; +import org.sleuthkit.autopsy.coreutils.PlatformUtil; import org.sleuthkit.autopsy.ingest.IngestManager; +import org.sleuthkit.autopsy.keywordsearch.KeywordSearchIngestModule.UpdateFrequency; /** * General, not per list, keyword search configuration and status display widget @@ -51,6 +53,31 @@ private void activateWidgets() { boolean ingestRunning = IngestManager.getInstance().isIngestRunning(); ingestWarningLabel.setVisible(ingestRunning); skipNSRLCheckBox.setEnabled(!ingestRunning); + setTimeSettingEnabled(!ingestRunning); + + final UpdateFrequency curFreq = KeywordSearchSettings.getUpdateFrequency(); + switch (curFreq) { + case FAST: + timeRadioButton1.setSelected(true); + break; + case AVG: + timeRadioButton2.setSelected(true); + break; + case SLOW: + timeRadioButton3.setSelected(true); + break; + case SLOWEST: + timeRadioButton4.setSelected(true); + break; + case NONE: + timeRadioButton5.setSelected(true); + break; + case DEFAULT: + default: + // default value + timeRadioButton3.setSelected(true); + break; + } } /** @@ -62,6 +89,7 @@ private void activateWidgets() { // <editor-fold defaultstate="collapsed" desc="Generated Code">//GEN-BEGIN:initComponents private void initComponents() { + timeGroup = new javax.swing.ButtonGroup(); skipNSRLCheckBox = new javax.swing.JCheckBox(); filesIndexedLabel = new javax.swing.JLabel(); filesIndexedValue = new javax.swing.JLabel(); @@ -71,7 +99,13 @@ private void initComponents() { informationLabel = new javax.swing.JLabel(); settingsSeparator = new javax.swing.JSeparator(); informationSeparator = new javax.swing.JSeparator(); + frequencyLabel = new javax.swing.JLabel(); + timeRadioButton1 = new javax.swing.JRadioButton(); + timeRadioButton2 = new javax.swing.JRadioButton(); + timeRadioButton3 = new javax.swing.JRadioButton(); + timeRadioButton4 = new javax.swing.JRadioButton(); showSnippetsCB = new javax.swing.JCheckBox(); + timeRadioButton5 = new javax.swing.JRadioButton(); ingestWarningLabel = new javax.swing.JLabel(); skipNSRLCheckBox.setText(org.openide.util.NbBundle.getMessage(KeywordSearchGlobalSearchSettingsPanel.class, "KeywordSearchGlobalSearchSettingsPanel.skipNSRLCheckBox.text")); // NOI18N @@ -94,6 +128,40 @@ public void actionPerformed(java.awt.event.ActionEvent evt) { informationLabel.setText(org.openide.util.NbBundle.getMessage(KeywordSearchGlobalSearchSettingsPanel.class, "KeywordSearchGlobalSearchSettingsPanel.informationLabel.text")); // NOI18N + frequencyLabel.setText(org.openide.util.NbBundle.getMessage(KeywordSearchGlobalSearchSettingsPanel.class, "KeywordSearchGlobalSearchSettingsPanel.frequencyLabel.text")); // NOI18N + + timeRadioButton1.setText(org.openide.util.NbBundle.getMessage(KeywordSearchGlobalSearchSettingsPanel.class, "KeywordSearchGlobalSearchSettingsPanel.timeRadioButton1.text")); // NOI18N + timeRadioButton1.setToolTipText(org.openide.util.NbBundle.getMessage(KeywordSearchGlobalSearchSettingsPanel.class, "KeywordSearchGlobalSearchSettingsPanel.timeRadioButton1.toolTipText")); // NOI18N + timeRadioButton1.addActionListener(new java.awt.event.ActionListener() { + public void actionPerformed(java.awt.event.ActionEvent evt) { + timeRadioButton1ActionPerformed(evt); + } + }); + + timeRadioButton2.setText(org.openide.util.NbBundle.getMessage(KeywordSearchGlobalSearchSettingsPanel.class, "KeywordSearchGlobalSearchSettingsPanel.timeRadioButton2.text")); // NOI18N + timeRadioButton2.setToolTipText(org.openide.util.NbBundle.getMessage(KeywordSearchGlobalSearchSettingsPanel.class, "KeywordSearchGlobalSearchSettingsPanel.timeRadioButton2.toolTipText")); // NOI18N + timeRadioButton2.addActionListener(new java.awt.event.ActionListener() { + public void actionPerformed(java.awt.event.ActionEvent evt) { + timeRadioButton2ActionPerformed(evt); + } + }); + + timeRadioButton3.setText(org.openide.util.NbBundle.getMessage(KeywordSearchGlobalSearchSettingsPanel.class, "KeywordSearchGlobalSearchSettingsPanel.timeRadioButton3.text")); // NOI18N + timeRadioButton3.setToolTipText(org.openide.util.NbBundle.getMessage(KeywordSearchGlobalSearchSettingsPanel.class, "KeywordSearchGlobalSearchSettingsPanel.timeRadioButton3.toolTipText")); // NOI18N + timeRadioButton3.addActionListener(new java.awt.event.ActionListener() { + public void actionPerformed(java.awt.event.ActionEvent evt) { + timeRadioButton3ActionPerformed(evt); + } + }); + + timeRadioButton4.setText(org.openide.util.NbBundle.getMessage(KeywordSearchGlobalSearchSettingsPanel.class, "KeywordSearchGlobalSearchSettingsPanel.timeRadioButton4.text_1")); // NOI18N + timeRadioButton4.setToolTipText(org.openide.util.NbBundle.getMessage(KeywordSearchGlobalSearchSettingsPanel.class, "KeywordSearchGlobalSearchSettingsPanel.timeRadioButton4.toolTipText")); // NOI18N + timeRadioButton4.addActionListener(new java.awt.event.ActionListener() { + public void actionPerformed(java.awt.event.ActionEvent evt) { + timeRadioButton4ActionPerformed(evt); + } + }); + showSnippetsCB.setText(org.openide.util.NbBundle.getMessage(KeywordSearchGlobalSearchSettingsPanel.class, "KeywordSearchGlobalSearchSettingsPanel.showSnippetsCB.text")); // NOI18N showSnippetsCB.addActionListener(new java.awt.event.ActionListener() { public void actionPerformed(java.awt.event.ActionEvent evt) { @@ -101,6 +169,14 @@ public void actionPerformed(java.awt.event.ActionEvent evt) { } }); + timeRadioButton5.setText(org.openide.util.NbBundle.getMessage(KeywordSearchGlobalSearchSettingsPanel.class, "KeywordSearchGlobalSearchSettingsPanel.timeRadioButton5.text")); // NOI18N + timeRadioButton5.setToolTipText(org.openide.util.NbBundle.getMessage(KeywordSearchGlobalSearchSettingsPanel.class, "KeywordSearchGlobalSearchSettingsPanel.timeRadioButton5.toolTipText")); // NOI18N + timeRadioButton5.addActionListener(new java.awt.event.ActionListener() { + public void actionPerformed(java.awt.event.ActionEvent evt) { + timeRadioButton5ActionPerformed(evt); + } + }); + ingestWarningLabel.setIcon(new javax.swing.ImageIcon(getClass().getResource("/org/sleuthkit/autopsy/modules/hashdatabase/warning16.png"))); // NOI18N ingestWarningLabel.setText(org.openide.util.NbBundle.getMessage(KeywordSearchGlobalSearchSettingsPanel.class, "KeywordSearchGlobalSearchSettingsPanel.ingestWarningLabel.text")); // NOI18N @@ -131,11 +207,19 @@ public void actionPerformed(java.awt.event.ActionEvent evt) { .addComponent(filesIndexedLabel) .addGap(18, 18, 18) .addComponent(filesIndexedValue)) + .addComponent(frequencyLabel) .addGroup(layout.createSequentialGroup() .addComponent(chunksLabel) .addGap(18, 18, 18) - .addComponent(chunksValLabel))) - .addGap(132, 132, 132))) + .addComponent(chunksValLabel)) + .addGroup(layout.createSequentialGroup() + .addGap(16, 16, 16) + .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.LEADING) + .addComponent(timeRadioButton2) + .addComponent(timeRadioButton1) + .addComponent(timeRadioButton3) + .addComponent(timeRadioButton4) + .addComponent(timeRadioButton5)))))) .addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)) .addGroup(layout.createSequentialGroup() .addComponent(settingsLabel) @@ -157,7 +241,19 @@ public void actionPerformed(java.awt.event.ActionEvent evt) { .addComponent(skipNSRLCheckBox) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) .addComponent(showSnippetsCB) - .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addComponent(frequencyLabel) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addComponent(timeRadioButton1) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addComponent(timeRadioButton2) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addComponent(timeRadioButton3) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addComponent(timeRadioButton4) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) + .addComponent(timeRadioButton5) + .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.RELATED) .addGroup(layout.createParallelGroup(javax.swing.GroupLayout.Alignment.TRAILING) .addComponent(informationLabel) .addComponent(informationSeparator, javax.swing.GroupLayout.PREFERRED_SIZE, 7, javax.swing.GroupLayout.PREFERRED_SIZE)) @@ -171,10 +267,14 @@ public void actionPerformed(java.awt.event.ActionEvent evt) { .addComponent(chunksValLabel)) .addPreferredGap(javax.swing.LayoutStyle.ComponentPlacement.UNRELATED) .addComponent(ingestWarningLabel) - .addContainerGap(151, Short.MAX_VALUE)) + .addContainerGap(javax.swing.GroupLayout.DEFAULT_SIZE, Short.MAX_VALUE)) ); }// </editor-fold>//GEN-END:initComponents + private void timeRadioButton5ActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_timeRadioButton5ActionPerformed + firePropertyChange(OptionsPanelController.PROP_CHANGED, null, null); + }//GEN-LAST:event_timeRadioButton5ActionPerformed + private void skipNSRLCheckBoxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_skipNSRLCheckBoxActionPerformed firePropertyChange(OptionsPanelController.PROP_CHANGED, null, null); }//GEN-LAST:event_skipNSRLCheckBoxActionPerformed @@ -183,11 +283,28 @@ private void showSnippetsCBActionPerformed(java.awt.event.ActionEvent evt) {//GE firePropertyChange(OptionsPanelController.PROP_CHANGED, null, null); }//GEN-LAST:event_showSnippetsCBActionPerformed + private void timeRadioButton1ActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_timeRadioButton1ActionPerformed + firePropertyChange(OptionsPanelController.PROP_CHANGED, null, null); + }//GEN-LAST:event_timeRadioButton1ActionPerformed + + private void timeRadioButton2ActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_timeRadioButton2ActionPerformed + firePropertyChange(OptionsPanelController.PROP_CHANGED, null, null); + }//GEN-LAST:event_timeRadioButton2ActionPerformed + + private void timeRadioButton3ActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_timeRadioButton3ActionPerformed + firePropertyChange(OptionsPanelController.PROP_CHANGED, null, null); + }//GEN-LAST:event_timeRadioButton3ActionPerformed + + private void timeRadioButton4ActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_timeRadioButton4ActionPerformed + firePropertyChange(OptionsPanelController.PROP_CHANGED, null, null); + }//GEN-LAST:event_timeRadioButton4ActionPerformed + // Variables declaration - do not modify//GEN-BEGIN:variables private javax.swing.JLabel chunksLabel; private javax.swing.JLabel chunksValLabel; private javax.swing.JLabel filesIndexedLabel; private javax.swing.JLabel filesIndexedValue; + private javax.swing.JLabel frequencyLabel; private javax.swing.JLabel informationLabel; private javax.swing.JSeparator informationSeparator; private javax.swing.JLabel ingestWarningLabel; @@ -195,11 +312,18 @@ private void showSnippetsCBActionPerformed(java.awt.event.ActionEvent evt) {//GE private javax.swing.JSeparator settingsSeparator; private javax.swing.JCheckBox showSnippetsCB; private javax.swing.JCheckBox skipNSRLCheckBox; + private javax.swing.ButtonGroup timeGroup; + private javax.swing.JRadioButton timeRadioButton1; + private javax.swing.JRadioButton timeRadioButton2; + private javax.swing.JRadioButton timeRadioButton3; + private javax.swing.JRadioButton timeRadioButton4; + private javax.swing.JRadioButton timeRadioButton5; // End of variables declaration//GEN-END:variables @Override public void store() { KeywordSearchSettings.setSkipKnown(skipNSRLCheckBox.isSelected()); + KeywordSearchSettings.setUpdateFrequency(getSelectedTimeValue()); KeywordSearchSettings.setShowSnippets(showSnippetsCB.isSelected()); } @@ -208,10 +332,40 @@ public void load() { activateWidgets(); } + private void setTimeSettingEnabled(boolean enabled) { + timeRadioButton1.setEnabled(enabled); + timeRadioButton2.setEnabled(enabled); + timeRadioButton3.setEnabled(enabled); + timeRadioButton4.setEnabled(enabled); + timeRadioButton5.setEnabled(enabled); + frequencyLabel.setEnabled(enabled); + } + + private UpdateFrequency getSelectedTimeValue() { + if (timeRadioButton1.isSelected()) { + return UpdateFrequency.FAST; + } else if (timeRadioButton2.isSelected()) { + return UpdateFrequency.AVG; + } else if (timeRadioButton3.isSelected()) { + return UpdateFrequency.SLOW; + } else if (timeRadioButton4.isSelected()) { + return UpdateFrequency.SLOWEST; + } else if (timeRadioButton5.isSelected()) { + return UpdateFrequency.NONE; + } + return UpdateFrequency.DEFAULT; + } + @NbBundle.Messages({"KeywordSearchGlobalSearchSettingsPanel.customizeComponents.windowsOCR=Enable Optical Character Recognition (OCR) (Requires Windows 64-bit)", "KeywordSearchGlobalSearchSettingsPanel.customizeComponents.windowsLimitedOCR=Only process images which are over 100KB in size or extracted from a document. (Beta) (Requires Windows 64-bit)"}) private void customizeComponents() { + timeGroup.add(timeRadioButton1); + timeGroup.add(timeRadioButton2); + timeGroup.add(timeRadioButton3); + timeGroup.add(timeRadioButton4); + timeGroup.add(timeRadioButton5); + this.skipNSRLCheckBox.setSelected(KeywordSearchSettings.getSkipKnown()); try { diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java index 9dcd3984dd3f0fc74a084e8de0545aec841f8776..922a0c0e5f1cd2b4ed1034dffd67a3eeaa6d94be 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchIngestModule.java @@ -1,7 +1,7 @@ /* * Autopsy Forensic Browser * - * Copyright 2011-2022 Basis Technology Corp. + * Copyright 2011-2021 Basis Technology Corp. * Contact: carrier <at> sleuthkit <dot> org * * Licensed under the Apache License, Version 2.0 (the "License"); @@ -170,6 +170,24 @@ enum StringsExtractOptions { EXTRACT_UTF8, ///< extract UTF8 text, true/false }; + enum UpdateFrequency { + + FAST(20), + AVG(10), + SLOW(5), + SLOWEST(1), + NONE(Integer.MAX_VALUE), + DEFAULT(5); + private final int time; + + UpdateFrequency(int time) { + this.time = time; + } + + int getTime() { + return time; + } + }; private static final Logger logger = Logger.getLogger(KeywordSearchIngestModule.class.getName()); private final IngestServices services = IngestServices.getInstance(); private Ingester ingester = null; @@ -178,6 +196,7 @@ enum StringsExtractOptions { //only search images from current ingest, not images previously ingested/indexed //accessed read-only by searcher thread + private boolean startedSearching = false; private Lookup stringsExtractionContext; private final KeywordSearchJobSettings settings; private boolean initialized = false; @@ -386,6 +405,16 @@ public ProcessResult process(AbstractFile abstractFile) { } indexer.indexAndSearchFile(extractorOpt, abstractFile, mimeType, true); + // Start searching if it hasn't started already + if (!startedSearching) { + if (context.fileIngestIsCancelled()) { + return ProcessResult.OK; + } + List<String> keywordListNames = settings.getNamesOfEnabledKeyWordLists(); + IngestSearchRunner.getInstance().startJob(context, keywordListNames); + startedSearching = true; + } + return ProcessResult.OK; } @@ -402,11 +431,15 @@ public void shutDown() { } if (context.fileIngestIsCancelled()) { - logger.log(Level.INFO, "Keyword search ingest module instance {0} stopping due to ingest cancellation", instanceNum); //NON-NLS + logger.log(Level.INFO, "Keyword search ingest module instance {0} stopping search job due to ingest cancellation", instanceNum); //NON-NLS + IngestSearchRunner.getInstance().stopJob(jobId); cleanup(); return; } + // Remove from the search list and trigger final commit and final search + IngestSearchRunner.getInstance().endJob(jobId); + // We only need to post the summary msg from the last module per job if (refCounter.decrementAndGet(jobId) == 0) { try { diff --git a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchSettings.java b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchSettings.java index 0087c7d05e01ebe7d31df0ad337ce08cf12c8f7e..b7057a2c89d0061bdd4d3ea4954d2a6952325760 100644 --- a/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchSettings.java +++ b/KeywordSearch/src/org/sleuthkit/autopsy/keywordsearch/KeywordSearchSettings.java @@ -29,6 +29,7 @@ import org.sleuthkit.autopsy.coreutils.StringExtract; import org.sleuthkit.autopsy.coreutils.StringExtract.StringExtractUnicodeTable.SCRIPT; import org.sleuthkit.autopsy.keywordsearch.KeywordSearchIngestModule.StringsExtractOptions; +import org.sleuthkit.autopsy.keywordsearch.KeywordSearchIngestModule.UpdateFrequency; //This file contains constants and settings for KeywordSearch class KeywordSearchSettings { @@ -45,9 +46,34 @@ class KeywordSearchSettings { static final boolean LIMITED_OCR_ENABLED_DEFAULT = false; private static boolean skipKnown = true; private static final Logger logger = Logger.getLogger(KeywordSearchSettings.class.getName()); + private static UpdateFrequency UpdateFreq = UpdateFrequency.DEFAULT; private static List<StringExtract.StringExtractUnicodeTable.SCRIPT> stringExtractScripts = new ArrayList<>(); private static Map<String, String> stringExtractOptions = new HashMap<>(); + /** + * Gets the update Frequency from KeywordSearch_Options.properties + * + * @return KeywordSearchIngestModule's update frequency + */ + static UpdateFrequency getUpdateFrequency() { + if (ModuleSettings.getConfigSetting(PROPERTIES_OPTIONS, "UpdateFrequency") != null) { //NON-NLS + return UpdateFrequency.valueOf(ModuleSettings.getConfigSetting(PROPERTIES_OPTIONS, "UpdateFrequency")); //NON-NLS + } + //if it failed, return the default/last known value + logger.log(Level.WARNING, "Could not read property for UpdateFrequency, returning backup value."); //NON-NLS + return UpdateFrequency.DEFAULT; + } + + /** + * Sets the update frequency and writes to KeywordSearch_Options.properties + * + * @param freq Sets KeywordSearchIngestModule to this value. + */ + static void setUpdateFrequency(UpdateFrequency freq) { + ModuleSettings.setConfigSetting(PROPERTIES_OPTIONS, "UpdateFrequency", freq.name()); //NON-NLS + UpdateFreq = freq; + } + /** * Sets whether or not to skip adding known good files to the search during * index. @@ -217,6 +243,11 @@ static void setDefaults() { logger.log(Level.INFO, "No configuration for NSRL found, generating default..."); //NON-NLS KeywordSearchSettings.setSkipKnown(true); } + //setting default Update Frequency + if (!ModuleSettings.settingExists(KeywordSearchSettings.PROPERTIES_OPTIONS, "UpdateFrequency")) { //NON-NLS + logger.log(Level.INFO, "No configuration for Update Frequency found, generating default..."); //NON-NLS + KeywordSearchSettings.setUpdateFrequency(UpdateFrequency.DEFAULT); + } //setting default Extract UTF8 if (!ModuleSettings.settingExists(KeywordSearchSettings.PROPERTIES_OPTIONS, StringsExtractOptions.EXTRACT_UTF8.toString())) { logger.log(Level.INFO, "No configuration for UTF8 found, generating default..."); //NON-NLS