diff --git a/Core/src/com/basistech/df/cybertriage/autopsy/ctoptions/ctcloud/CTLicensePersistence.java b/Core/src/com/basistech/df/cybertriage/autopsy/ctoptions/ctcloud/CTLicensePersistence.java
index 67f19201108c7fa305a9cbcfb2128196570c18eb..e95d9e72795c57d1a37d206fc19864681ada9b3e 100644
--- a/Core/src/com/basistech/df/cybertriage/autopsy/ctoptions/ctcloud/CTLicensePersistence.java
+++ b/Core/src/com/basistech/df/cybertriage/autopsy/ctoptions/ctcloud/CTLicensePersistence.java
@@ -92,19 +92,15 @@ public synchronized Optional<LicenseInfo> loadLicenseInfo() {
});
}
- public synchronized boolean saveMalwareSettings(MalwareIngestSettings malwareIngestSettings) {
+ public synchronized boolean saveMalwareSettings(MalwareIngestSettings malwareIngestSettings) {
if (malwareIngestSettings != null) {
File settingsFile = getMalwareIngestFile();
try {
settingsFile.getParentFile().mkdirs();
- if (licenseResponse != null) {
- objectMapper.writeValue(licenseFile, licenseResponse);
- } else if (licenseFile.exists()) {
- Files.delete(licenseFile.toPath());
- }
+ objectMapper.writeValue(settingsFile, malwareIngestSettings);
return true;
} catch (IOException ex) {
- logger.log(Level.WARNING, "There was an error writing CyberTriage license to file: " + licenseFile.getAbsolutePath(), ex);
+ logger.log(Level.WARNING, "There was an error writing malware ingest settings to file: " + settingsFile.getAbsolutePath(), ex);
}
}
@@ -112,19 +108,23 @@ public synchronized boolean saveMalwareSettings(MalwareIngestSettings malwareIng
}
public synchronized MalwareIngestSettings loadMalwareIngestSettings() {
- Optional<LicenseResponse> toRet = Optional.empty();
- File licenseFile = getCTLicenseFile();
- if (licenseFile.exists() && licenseFile.isFile()) {
+ MalwareIngestSettings settings = null;
+ File settingsFile = getMalwareIngestFile();
+ if (settingsFile.exists() && settingsFile.isFile()) {
try {
- toRet = Optional.ofNullable(objectMapper.readValue(licenseFile, LicenseResponse.class));
+ settings = objectMapper.readValue(settingsFile, MalwareIngestSettings.class);
} catch (IOException ex) {
- logger.log(Level.WARNING, "There was an error reading CyberTriage license to file: " + licenseFile.getAbsolutePath(), ex);
+ logger.log(Level.WARNING, "There was an error reading malware ingest settings from file: " + settingsFile.getAbsolutePath(), ex);
}
}
+
+ if (settings == null) {
+ settings = new MalwareIngestSettings();
+ }
- return toRet;
+ return settings;
}
-
+
private File getCTLicenseFile() {
return Paths.get(PlatformUtil.getModuleConfigDirectory(), CT_SETTINGS_DIR, CT_LICENSE_FILENAME).toFile();
}
diff --git a/Core/src/com/basistech/df/cybertriage/autopsy/ctoptions/ctcloud/CTMalwareScannerOptionsPanel.java b/Core/src/com/basistech/df/cybertriage/autopsy/ctoptions/ctcloud/CTMalwareScannerOptionsPanel.java
index 94ba8fb948336695776cd42b1d8801b71506a743..b380dded422c5b61f8bde48d57c7d3b3deaac7d3 100644
--- a/Core/src/com/basistech/df/cybertriage/autopsy/ctoptions/ctcloud/CTMalwareScannerOptionsPanel.java
+++ b/Core/src/com/basistech/df/cybertriage/autopsy/ctoptions/ctcloud/CTMalwareScannerOptionsPanel.java
@@ -111,6 +111,7 @@ public void componentShown(ComponentEvent e) {
@Override
public synchronized void saveSettings() {
ctPersistence.saveLicenseResponse(getLicenseInfo());
+ ctPersistence.saveMalwareSettings(getIngestSettings());
}
@Override
@@ -127,11 +128,26 @@ public synchronized void loadSettings() {
if (licenseInfo != null) {
loadMalwareScansInfo(licenseInfo);
}
+
+ MalwareIngestSettings ingestSettings = ctPersistence.loadMalwareIngestSettings();
+ setIngestSettings(ingestSettings);
}
private synchronized LicenseResponse getLicenseInfo() {
return this.licenseInfo == null ? null : this.licenseInfo.getLicenseResponse();
}
+
+ private MalwareIngestSettings getIngestSettings() {
+ return new MalwareIngestSettings()
+ .setUploadFiles(this.fileUploadCheckbox.isSelected());
+ }
+
+ private void setIngestSettings(MalwareIngestSettings ingestSettings) {
+ if (ingestSettings == null) {
+ ingestSettings = new MalwareIngestSettings();
+ }
+ this.fileUploadCheckbox.setSelected(ingestSettings.isUploadFiles());
+ }
private synchronized void setLicenseDisplay(LicenseInfo licenseInfo, String licenseMessage) {
this.licenseInfo = licenseInfo;
@@ -411,7 +427,7 @@ private void licenseInfoAddButtonActionPerformed(java.awt.event.ActionEvent evt)
}//GEN-LAST:event_licenseInfoAddButtonActionPerformed
private void fileUploadCheckboxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_fileUploadCheckboxActionPerformed
- // TODO add your handling code here:
+ this.firePropertyChange(OptionsPanelController.PROP_CHANGED, null, null);
}//GEN-LAST:event_fileUploadCheckboxActionPerformed
@NbBundle.Messages({
diff --git a/Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/MalwareScanIngestModule.java b/Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/MalwareScanIngestModule.java
index dd8e45a085f2250dbdd42bd8197644f2503e7fe3..a509310391789c7bfd9648a2e71b848ce4150ac9 100644
--- a/Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/MalwareScanIngestModule.java
+++ b/Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/MalwareScanIngestModule.java
@@ -40,6 +40,7 @@
import java.util.stream.Stream;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
+import org.apache.curator.shaded.com.google.common.collect.Lists;
import org.openide.util.NbBundle.Messages;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.coreutils.Logger;
@@ -96,6 +97,9 @@ private static class SharedProcessing {
private static final long MIN_UPLOAD_SIZE = 1;
private static final long MAX_UPLOAD_SIZE = 1_000_000_000;
+ private static final int NUM_FILE_UPLOAD_RETRIES = 60 * 5;
+ private static final long FILE_UPLOAD_RETRY_SLEEP_MILLIS = 60 * 1000;
+
private static final Set<String> EXECUTABLE_MIME_TYPES = Stream.of(
"application/x-bat",//NON-NLS
@@ -128,6 +132,7 @@ private static class SharedProcessing {
private BlackboardArtifact.Type malwareType = null;
private long dsId = 0;
private long ingestJobId = 0;
+ private boolean uploadUnknownFiles = false;
@Messages({
"MalwareScanIngestModule_ShareProcessing_lowLimitWarning_title=Hash Lookups Low",
@@ -186,6 +191,7 @@ synchronized void startUp(IngestJobContext context) throws IngestModuleException
dsId = context.getDataSource().getId();
ingestJobId = context.getJobId();
licenseInfo = licenseInfoOpt.get();
+ uploadUnknownFiles = ctSettingsPersistence.loadMalwareIngestSettings().isUploadFiles();
// set run state to initialized
runState = RunState.STARTED_UP;
@@ -201,55 +207,6 @@ private static long remaining(Long limit, Long used) {
return limit - used;
}
- private boolean isUnknown(CTCloudBean cloudBean) {
- return cloudBean != null
- && cloudBean.getMalwareResult() != null
- && cloudBean.getMalwareResult().getStatus() == MalwareResultBean.Status.NOT_FOUND;
- }
-
- private boolean isUploadable(AbstractFile af) {
- long size = af.getSize();
- return size >= MIN_UPLOAD_SIZE && size <= MAX_UPLOAD_SIZE;
- }
-
- private boolean uploadFile(SleuthkitCase skCase, DecryptedLicenseResponse decrypted, CTCloudBean cloudBean, long objId) throws CTCloudException, TskCoreException {
- if (!isUnknown(cloudBean)) {
- return false;
- }
-
- AbstractFile af = skCase.getAbstractFileById(objId);
- if (af == null) {
- return false;
- }
-
- if (!isUploadable(af)) {
- return false;
- }
-
- // get auth token / file upload url
- AuthTokenResponse authTokenResponse = ctApiDAO.getAuthToken(decrypted, true);
- if (StringUtils.isBlank(authTokenResponse.getFileUploadUrl())) {
- throw new CTCloudException(CTCloudException.ErrorCode.NETWORK_ERROR);
- }
-
- // upload bytes
- ReadContentInputStream fileInputStream = new ReadContentInputStream(af);
- ctApiDAO.uploadFile(authTokenResponse.getFileUploadUrl(), af.getName(), fileInputStream);
-
- // upload metadata
- MetadataUploadRequest metaRequest = new MetadataUploadRequest()
- .setCreatedDate(af.getCrtime())
- .setFilePath(af.getUniquePath())
- .setFileSizeBytes(af.getSize())
- .setFileUploadUrl(authTokenResponse.getFileUploadUrl())
- .setMd5(af.getMd5Hash())
- .setSha1(af.getSha1Hash())
- .setSha256(af.getSha256Hash());
-
- ctApiDAO.uploadMeta(new AuthenticatedRequestData(decrypted, authTokenResponse), metaRequest);
- return true;
- }
-
@Messages({
"MalwareScanIngestModule_ShareProcessing_batchTimeout_title=Batch Processing Timeout",
"MalwareScanIngestModule_ShareProcessing_batchTimeout_desc=Batch processing timed out"
@@ -391,6 +348,82 @@ private String sanitizedMd5(String orig) {
return StringUtils.defaultString(orig).trim().toLowerCase();
}
+ private boolean isUnknown(CTCloudBean cloudBean) {
+ return cloudBean != null
+ && cloudBean.getMalwareResult() != null
+ && cloudBean.getMalwareResult().getStatus() == MalwareResultBean.Status.NOT_FOUND;
+ }
+
+ private boolean isUploadable(AbstractFile af) {
+ long size = af.getSize();
+ return size >= MIN_UPLOAD_SIZE && size <= MAX_UPLOAD_SIZE;
+ }
+
+ private boolean uploadFile(CTCloudBean cloudBean, long objId) throws CTCloudException, TskCoreException {
+ if (!uploadUnknownFiles) {
+ return false;
+ }
+
+ if (!isUnknown(cloudBean)) {
+ return false;
+ }
+
+ AbstractFile af = skCase.getAbstractFileById(objId);
+ if (af == null) {
+ return false;
+ }
+
+ if (!isUploadable(af)) {
+ return false;
+ }
+
+ // get auth token / file upload url
+ AuthTokenResponse authTokenResponse = ctApiDAO.getAuthToken(decrypted, true);
+ if (StringUtils.isBlank(authTokenResponse.getFileUploadUrl())) {
+ throw new CTCloudException(CTCloudException.ErrorCode.NETWORK_ERROR);
+ } else if (remaining(authTokenResponse.getFileUploadLimit(), authTokenResponse.getFileUploadCount()) <= 0) {
+ // don't proceed with upload if reached limit
+ uploadUnknownFiles = false;
+ return false;
+ }
+
+ // upload bytes
+ ReadContentInputStream fileInputStream = new ReadContentInputStream(af);
+ ctApiDAO.uploadFile(authTokenResponse.getFileUploadUrl(), af.getName(), fileInputStream);
+
+ // upload metadata
+ MetadataUploadRequest metaRequest = new MetadataUploadRequest()
+ .setCreatedDate(af.getCrtime())
+ .setFilePath(af.getUniquePath())
+ .setFileSizeBytes(af.getSize())
+ .setFileUploadUrl(authTokenResponse.getFileUploadUrl())
+ .setMd5(af.getMd5Hash())
+ .setSha1(af.getSha1Hash())
+ .setSha256(af.getSha256Hash());
+
+ ctApiDAO.uploadMeta(new AuthenticatedRequestData(decrypted, authTokenResponse), metaRequest);
+ return true;
+ }
+
+ private boolean getUploadedFileResults(Map<String, List<Long>> md5objIdMapping) {
+ Map<String, List<Long>> remaining = new HashMap<>(md5objIdMapping);
+
+ for (int retry = 0; retry < NUM_FILE_UPLOAD_RETRIES; retry++) {
+ List<List<String>> md5Batches = Lists.partition(new ArrayList<>(remaining.keySet()), BATCH_SIZE);
+ for (List<String> batch : md5Batches) {
+ // TODO query and capture still unknown
+ }
+
+ if (remaining.isEmpty()) {
+ return true;
+ }
+
+
+ Thread.sleep(FILE_UPLOAD_RETRY_SLEEP_MILLIS);
+ }
+
+ }
+
@Messages({
"MalwareScanIngestModule_SharedProcessing_createAnalysisResult_Yes=YES",
"MalwareScanIngestModule_SharedProcessing_createAnalysisResult_No=NO"