diff --git a/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/CTCloudHttpClient.java b/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/CTCloudHttpClient.java
index 0fa9feda54ca30a8c4d04fc73844a54922967c32..b5d62f91de57431371dd4e3da4037b00332d4832 100644
--- a/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/CTCloudHttpClient.java
+++ b/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/CTCloudHttpClient.java
@@ -62,6 +62,7 @@
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.client.SystemDefaultCredentialsProvider;
import org.apache.http.impl.client.WinHttpClients;
+import org.sleuthkit.autopsy.coreutils.Version;
/**
* Actually makes the http requests to CT cloud.
@@ -70,6 +71,7 @@ public class CTCloudHttpClient {
private static final CTCloudHttpClient instance = new CTCloudHttpClient();
private static final Logger LOGGER = Logger.getLogger(CTCloudHttpClient.class.getName());
+ private static final String HOST_URL = Version.getBuildType() == Version.Type.RELEASE ? Constants.CT_CLOUD_SERVER : Constants.CT_CLOUD_DEV_SERVER;
private static final List<String> DEFAULT_SCHEME_PRIORITY
= new ArrayList<>(Arrays.asList(
@@ -129,7 +131,7 @@ public <O> O doPost(String urlPath, Object jsonBody, Class<O> classType) throws
}
public <O> O doPost(String urlPath, Map<String, String> urlReqParams, Object jsonBody, Class<O> classType) throws CTCloudException {
- String url = Constants.CT_CLOUD_SERVER + urlPath;
+ String url = HOST_URL + urlPath;
try {
LOGGER.log(Level.INFO, "initiating http connection to ctcloud server");
diff --git a/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/Constants.java b/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/Constants.java
index fa35dd7dbb7ba70d895d3e3b1ac28ddd9bf0d64d..ddda5b45089223c343b5980f77921f5c0ec8e1e1 100644
--- a/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/Constants.java
+++ b/Core/src/com/basistech/df/cybertriage/autopsy/ctapi/Constants.java
@@ -34,8 +34,7 @@ final public class Constants {
public static final String CT_CLOUD_DEV_SERVER = "https://cyber-triage-dev.appspot.com";
- // TODO put back
- public static final String CT_CLOUD_SERVER = CT_CLOUD_DEV_SERVER; //"https://rep1.cybertriage.com";
+ public static final String CT_CLOUD_SERVER = "https://rep1.cybertriage.com";
/**
* Link to watch demo video
diff --git a/Core/src/com/basistech/df/cybertriage/autopsy/ctoptions/Bundle.properties b/Core/src/com/basistech/df/cybertriage/autopsy/ctoptions/Bundle.properties
index 68677cb3f7ad19dbf22a1fa4cd16abd470e36590..46f62fdf7a5185355907a8f9980343d9478c03de 100644
--- a/Core/src/com/basistech/df/cybertriage/autopsy/ctoptions/Bundle.properties
+++ b/Core/src/com/basistech/df/cybertriage/autopsy/ctoptions/Bundle.properties
@@ -1,8 +1,8 @@
# Click nbfs://nbhost/SystemFileSystem/Templates/Licenses/license-default.txt to change this license
# Click nbfs://nbhost/SystemFileSystem/Templates/Other/properties.properties to edit this template
-OptionsCategory_Name_CyberTriage=CyberTriage
-OptionsCategory_Keywords_CyberTriage=CyberTriage,Cyber,Triage
+OptionsCategory_Name_CyberTriage=Cyber Triage
+OptionsCategory_Keywords_CyberTriage=Cyber Triage,Cyber,Triage
LicenseDisclaimerPanel.disclaimer.text=<html>The Cyber Triage Malware Scanner module uses 40+ malware scanning engines to identify if Windows executables are malicious. It requires a non-free license to use.</html>
LicenseDisclaimerPanel.purchaseFromLabel.text=You can purchase a license from
LicenseDisclaimerPanel.link.text=<html><span style="color: blue; text-decoration: underline">https://cybertriage.com/autopsy-checkout</span></html>
diff --git a/Core/src/com/basistech/df/cybertriage/autopsy/ctoptions/Bundle.properties-MERGED b/Core/src/com/basistech/df/cybertriage/autopsy/ctoptions/Bundle.properties-MERGED
index 68677cb3f7ad19dbf22a1fa4cd16abd470e36590..46f62fdf7a5185355907a8f9980343d9478c03de 100644
--- a/Core/src/com/basistech/df/cybertriage/autopsy/ctoptions/Bundle.properties-MERGED
+++ b/Core/src/com/basistech/df/cybertriage/autopsy/ctoptions/Bundle.properties-MERGED
@@ -1,8 +1,8 @@
# Click nbfs://nbhost/SystemFileSystem/Templates/Licenses/license-default.txt to change this license
# Click nbfs://nbhost/SystemFileSystem/Templates/Other/properties.properties to edit this template
-OptionsCategory_Name_CyberTriage=CyberTriage
-OptionsCategory_Keywords_CyberTriage=CyberTriage,Cyber,Triage
+OptionsCategory_Name_CyberTriage=Cyber Triage
+OptionsCategory_Keywords_CyberTriage=Cyber Triage,Cyber,Triage
LicenseDisclaimerPanel.disclaimer.text=<html>The Cyber Triage Malware Scanner module uses 40+ malware scanning engines to identify if Windows executables are malicious. It requires a non-free license to use.</html>
LicenseDisclaimerPanel.purchaseFromLabel.text=You can purchase a license from
LicenseDisclaimerPanel.link.text=<html><span style="color: blue; text-decoration: underline">https://cybertriage.com/autopsy-checkout</span></html>
diff --git a/Core/src/com/basistech/df/cybertriage/autopsy/ctoptions/LicenseDisclaimerPanel.java b/Core/src/com/basistech/df/cybertriage/autopsy/ctoptions/LicenseDisclaimerPanel.java
index ffd3760e6fbeba02e8aba1808be4bd87b2e1841d..4299f02d620e69d06770ed0e5b98a326c58069fa 100644
--- a/Core/src/com/basistech/df/cybertriage/autopsy/ctoptions/LicenseDisclaimerPanel.java
+++ b/Core/src/com/basistech/df/cybertriage/autopsy/ctoptions/LicenseDisclaimerPanel.java
@@ -22,16 +22,18 @@
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
+import java.util.logging.Level;
import org.sleuthkit.autopsy.coreutils.Logger;
/**
* Disclaimer for license and place to purchase CT license.
*/
public class LicenseDisclaimerPanel extends javax.swing.JPanel {
+
private static final Logger LOGGER = Logger.getLogger(LicenseDisclaimerPanel.class.getName());
-
+
private static final String CHECKOUT_PAGE_URL = "https://cybertriage.com/autopsy-checkout";
-
+
/**
* Creates new form LicenseDisclaimerPanel
*/
@@ -116,9 +118,10 @@ private void linkMouseClicked(java.awt.event.MouseEvent evt) {//GEN-FIRST:event_
try {
Desktop.getDesktop().browse(new URI(CHECKOUT_PAGE_URL));
} catch (IOException | URISyntaxException e) {
- /* TODO: error handling */ }
+ LOGGER.log(Level.SEVERE, "Error opening link to: " + CHECKOUT_PAGE_URL, e);
+ }
} else {
- /* TODO: error handling */
+ LOGGER.log(Level.WARNING, "Desktop API is not supported. Link cannot be opened.");
}
}//GEN-LAST:event_linkMouseClicked
diff --git a/Core/src/com/basistech/df/cybertriage/autopsy/ctoptions/ctcloud/CTLicenseDialog.java b/Core/src/com/basistech/df/cybertriage/autopsy/ctoptions/ctcloud/CTLicenseDialog.java
index 11af572890df51719cdf8d22f263ba1a5b31da46..608ea6304006f80b9a14299975a12140e2cf7d6c 100644
--- a/Core/src/com/basistech/df/cybertriage/autopsy/ctoptions/ctcloud/CTLicenseDialog.java
+++ b/Core/src/com/basistech/df/cybertriage/autopsy/ctoptions/ctcloud/CTLicenseDialog.java
@@ -29,9 +29,7 @@
*/
public class CTLicenseDialog extends javax.swing.JDialog {
- //private static final Pattern LICENSE_PATTERN = Pattern.compile("^\\s*[0-9a-zA-Z]{8}\\-[0-9a-zA-Z]{4}\\-[0-9a-zA-Z]{4}\\-[0-9a-zA-Z]{4}\\-[0-9a-zA-Z]{12}\\s*$");
-
- private static final Pattern LICENSE_PATTERN = Pattern.compile("\\s*[a-zA-Z0-9\\-]+?\\s*");
+ private static final Pattern LICENSE_PATTERN = Pattern.compile("^\\s*[a-zA-Z0-9\\-]+?\\s*$");
private String licenseString = null;
/**
diff --git a/Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/BatchProcessor.java b/Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/BatchProcessor.java
index dc62f6a05efe55c65229e53a10aab877db488141..eab025a6416569645480ca93437cc353a22ed467 100644
--- a/Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/BatchProcessor.java
+++ b/Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/BatchProcessor.java
@@ -38,27 +38,18 @@
*/
public class BatchProcessor<T> {
- private final ExecutorService processingExecutorService = Executors.newSingleThreadExecutor();
+ private ExecutorService processingExecutorService = Executors.newSingleThreadExecutor();
private final BlockingQueue<T> batchingQueue;
- private final List<T> processingQueue;
private final int batchSize;
private final Consumer<List<T>> itemsConsumer;
- private final long millisTimeout;
+ private final long secondsTimeout;
- private Future<?> lastProcessingFuture = CompletableFuture.runAsync(() -> {
- });
-
- public BatchProcessor(int batchSize, long millisTimeout, Consumer<List<T>> itemsConsumer) {
+ public BatchProcessor(int batchSize, long secondsTimeout, Consumer<List<T>> itemsConsumer) {
this.batchingQueue = new LinkedBlockingQueue<>(batchSize);
- this.processingQueue = new ArrayList<>(batchSize);
this.batchSize = batchSize;
this.itemsConsumer = itemsConsumer;
- this.millisTimeout = millisTimeout;
- }
-
- public synchronized void clearCurrentBatch() {
- batchingQueue.clear();
+ this.secondsTimeout = secondsTimeout;
}
public synchronized void add(T item) throws InterruptedException {
@@ -68,40 +59,29 @@ public synchronized void add(T item) throws InterruptedException {
}
}
- public synchronized void flush(boolean blockUntilFinished) throws InterruptedException {
+ public synchronized void flushAndReset() throws InterruptedException {
+ // get any remaining
asyncProcessBatch();
- if (blockUntilFinished) {
- waitCurrentFuture();
- }
- }
-
- private synchronized void waitCurrentFuture() throws InterruptedException {
- synchronized (lastProcessingFuture) {
- if (!lastProcessingFuture.isDone()) {
- try {
- lastProcessingFuture.get(millisTimeout, TimeUnit.MILLISECONDS);
- } catch (ExecutionException | TimeoutException ex) {
- // ignore timeout
- }
- }
- }
+
+ // don't accept any new additions
+ processingExecutorService.shutdown();
+
+ // await termination
+ processingExecutorService.awaitTermination(secondsTimeout, TimeUnit.SECONDS);
+
+ // get new (not shut down executor)
+ processingExecutorService = Executors.newSingleThreadExecutor();
}
private synchronized void asyncProcessBatch() throws InterruptedException {
if (!batchingQueue.isEmpty()) {
- // wait for previous processing to finish
- waitCurrentFuture();
-
- // if 'andThen' doesn't run, clear the processing queue
- processingQueue.clear();
+ final List<T> processingList = new ArrayList<>();
// transfer batching queue to processing queue
- batchingQueue.drainTo(processingQueue);
+ batchingQueue.drainTo(processingList);
- // submit to processor and then clear processing queue
- lastProcessingFuture = processingExecutorService.submit(
- () -> itemsConsumer.andThen(processingQueue -> processingQueue.clear()).accept(processingQueue)
- );
+ // submit to be processed
+ processingExecutorService.submit(() -> itemsConsumer.accept(processingList));
}
}
diff --git a/Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/MalwareScanIngestModule.java b/Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/MalwareScanIngestModule.java
index a509310391789c7bfd9648a2e71b848ce4150ac9..e249b487d5d5972749f4979712666315f1cf9e39 100644
--- a/Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/MalwareScanIngestModule.java
+++ b/Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/MalwareScanIngestModule.java
@@ -28,6 +28,7 @@
import com.basistech.df.cybertriage.autopsy.ctapi.json.MalwareResultBean;
import com.basistech.df.cybertriage.autopsy.ctapi.json.MetadataUploadRequest;
import com.basistech.df.cybertriage.autopsy.ctoptions.ctcloud.CTLicensePersistence;
+import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -54,6 +55,9 @@
import org.sleuthkit.datamodel.Blackboard;
import org.sleuthkit.datamodel.BlackboardArtifact;
import org.sleuthkit.datamodel.ReadContentInputStream;
+import org.sleuthkit.datamodel.HashUtility;
+import org.sleuthkit.datamodel.HashUtility.HashResult;
+import org.sleuthkit.datamodel.HashUtility.HashType;
import org.sleuthkit.datamodel.Score;
import org.sleuthkit.datamodel.SleuthkitCase;
import org.sleuthkit.datamodel.TskCoreException;
@@ -89,8 +93,8 @@ private static class SharedProcessing {
// batch size of 200 files max
private static final int BATCH_SIZE = 200;
- // 3 minute timeout for an API request
- private static final long BATCH_MILLIS_TIMEOUT = 3 * 60 * 1000;
+ // 1 day timeout for all API requests
+ private static final long FLUSH_SECS_TIMEOUT = 24 * 60 * 60;
//minimum lookups left before issuing warning
private static final long LOW_LOOKUPS_REMAINING = 250;
@@ -119,7 +123,7 @@ private static class SharedProcessing {
private static final String MALWARE_CONFIG = "Cyber Triage Cloud";
private static final Logger logger = Logger.getLogger(MalwareScanIngestModule.class.getName());
- private final BatchProcessor<FileRecord> batchProcessor = new BatchProcessor<FileRecord>(BATCH_SIZE, BATCH_MILLIS_TIMEOUT, this::handleBatch);
+ private final BatchProcessor<FileRecord> batchProcessor = new BatchProcessor<FileRecord>(BATCH_SIZE, FLUSH_SECS_TIMEOUT, this::handleBatch);
private final CTLicensePersistence ctSettingsPersistence = CTLicensePersistence.getInstance();
private final CTApiDAO ctApiDAO = CTApiDAO.getInstance();
@@ -207,18 +211,54 @@ private static long remaining(Long limit, Long used) {
return limit - used;
}
+ private String getOrCalcHash(AbstractFile af) {
+ if (StringUtils.isNotBlank(af.getMd5Hash())) {
+ return af.getMd5Hash();
+ }
+
+ try {
+ List<HashResult> hashResults = HashUtility.calculateHashes(af, Collections.singletonList(HashType.MD5));
+ if (CollectionUtils.isNotEmpty(hashResults)) {
+ for (HashResult hashResult : hashResults) {
+ if (hashResult.getType() == HashType.MD5) {
+ return hashResult.getValue();
+ }
+ }
+ }
+ } catch (TskCoreException ex) {
+ logger.log(Level.WARNING,
+ MessageFormat.format("An error occurred while processing file name: {0} and obj id: {1}.",
+ af.getName(),
+ af.getId()),
+ ex);
+ }
+
+ return null;
+ }
+
@Messages({
"MalwareScanIngestModule_ShareProcessing_batchTimeout_title=Batch Processing Timeout",
"MalwareScanIngestModule_ShareProcessing_batchTimeout_desc=Batch processing timed out"
})
IngestModule.ProcessResult process(AbstractFile af) {
try {
- if (runState == RunState.STARTED_UP && af.getKnown() != TskData.FileKnown.KNOWN
- && EXECUTABLE_MIME_TYPES.contains(StringUtils.defaultString(fileTypeDetector.getMIMEType(af)).trim().toLowerCase())) {
- batchProcessor.add(new FileRecord(af.getId(), af.getMd5Hash()));
-
+ if (runState == RunState.STARTED_UP
+ && af.getKnown() != TskData.FileKnown.KNOWN
+ && EXECUTABLE_MIME_TYPES.contains(StringUtils.defaultString(fileTypeDetector.getMIMEType(af)).trim().toLowerCase())
+ && CollectionUtils.isEmpty(af.getAnalysisResults(malwareType))) {
+
+ String md5 = getOrCalcHash(af);
+ if (StringUtils.isNotBlank(md5)) {
+ batchProcessor.add(new FileRecord(af.getId(), md5));
+ }
}
return ProcessResult.OK;
+ } catch (TskCoreException ex) {
+ notifyWarning(
+ Bundle.MalwareScanIngestModule_SharedProcessing_generalProcessingError_title(),
+ Bundle.MalwareScanIngestModule_SharedProcessing_generalProcessingError_desc(),
+ ex);
+ return IngestModule.ProcessResult.ERROR;
} catch (InterruptedException ex) {
notifyWarning(
Bundle.MalwareScanIngestModule_ShareProcessing_batchTimeout_title(),
@@ -246,7 +286,7 @@ private void handleBatch(List<FileRecord> fileRecords) {
// create mapping of md5 to corresponding object ids as well as just the list of md5's
Map<String, List<Long>> md5ToObjId = new HashMap<>();
- List<String> md5Hashes = new ArrayList<>();
+
for (FileRecord fr : fileRecords) {
if (fr == null || StringUtils.isBlank(fr.getMd5hash()) || fr.getObjId() <= 0) {
continue;
@@ -257,9 +297,10 @@ private void handleBatch(List<FileRecord> fileRecords) {
.computeIfAbsent(sanitizedMd5, (k) -> new ArrayList<>())
.add(fr.getObjId());
- md5Hashes.add(sanitizedMd5);
}
+ List<String> md5Hashes = new ArrayList<>(md5ToObjId.keySet());
+
if (md5Hashes.isEmpty()) {
return;
}
@@ -279,13 +320,6 @@ private void handleBatch(List<FileRecord> fileRecords) {
return;
}
- // if the size of this batch will exceed limit, shrink list to limit and fail after processing
- boolean exceededScanLimit = false;
- if (remainingScans < md5Hashes.size()) {
- md5Hashes = md5Hashes.subList(0, (int) remainingScans);
- exceededScanLimit = true;
- }
-
// using auth token, get results
List<CTCloudBean> repResult = ctApiDAO.getReputationResults(
new AuthenticatedRequestData(licenseInfo.getDecryptedLicense(), authTokenResponse),
@@ -325,16 +359,6 @@ private void handleBatch(List<FileRecord> fileRecords) {
if (!CollectionUtils.isEmpty(createdArtifacts)) {
tskCase.getBlackboard().postArtifacts(createdArtifacts, Bundle.MalwareScanIngestModuleFactory_displayName(), ingestJobId);
}
-
- // if we only processed part of the batch, after processing, notify that we are out of scans.
- if (exceededScanLimit) {
- runState = RunState.DISABLED;
- notifyWarning(
- Bundle.MalwareScanIngestModule_SharedProcessing_exhaustedHashLookups_title(),
- Bundle.MalwareScanIngestModule_SharedProcessing_exhaustedHashLookups_desc(),
- null);
- return;
- }
}
} catch (Exception ex) {
notifyWarning(
@@ -467,7 +491,7 @@ synchronized void shutDown() {
// flush any remaining items
try {
- batchProcessor.flush(true);
+ batchProcessor.flushAndReset();
} catch (InterruptedException ex) {
notifyWarning(
Bundle.MalwareScanIngestModule_SharedProcessing_flushTimeout_title(),
@@ -476,7 +500,6 @@ synchronized void shutDown() {
} finally {
// set state to shut down and clear any remaining
runState = RunState.SHUT_DOWN;
- batchProcessor.clearCurrentBatch();
}
}