Skip to content
Snippets Groups Projects
Commit fe26a6e0 authored by Greg DiCristofaro's avatar Greg DiCristofaro
Browse files

updates

parent 5d581f11
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Core/src/com/basistech/df/cybertriage/autopsy/ctoptions/ctcloud/CTLicensePersistence.java
+ 14
− 14
View file @ fe26a6e0
......@@ -92,19 +92,15 @@ public synchronized Optional<LicenseInfo> loadLicenseInfo() {
});
}
public synchronized boolean saveMalwareSettings(MalwareIngestSettings malwareIngestSettings) {
public synchronized boolean saveMalwareSettings(MalwareIngestSettings malwareIngestSettings) {
if (malwareIngestSettings != null) {
File settingsFile = getMalwareIngestFile();
try {
settingsFile.getParentFile().mkdirs();
if (licenseResponse != null) {
objectMapper.writeValue(licenseFile, licenseResponse);
} else if (licenseFile.exists()) {
Files.delete(licenseFile.toPath());
}
objectMapper.writeValue(settingsFile, malwareIngestSettings);
return true;
} catch (IOException ex) {
logger.log(Level.WARNING, "There was an error writing CyberTriage license to file: " + licenseFile.getAbsolutePath(), ex);
logger.log(Level.WARNING, "There was an error writing malware ingest settings to file: " + settingsFile.getAbsolutePath(), ex);
}
}
......@@ -112,19 +108,23 @@ public synchronized boolean saveMalwareSettings(MalwareIngestSettings malwareIng
}
public synchronized MalwareIngestSettings loadMalwareIngestSettings() {
Optional<LicenseResponse> toRet = Optional.empty();
File licenseFile = getCTLicenseFile();
if (licenseFile.exists() && licenseFile.isFile()) {
MalwareIngestSettings settings = null;
File settingsFile = getMalwareIngestFile();
if (settingsFile.exists() && settingsFile.isFile()) {
try {
toRet = Optional.ofNullable(objectMapper.readValue(licenseFile, LicenseResponse.class));
settings = objectMapper.readValue(settingsFile, MalwareIngestSettings.class);
} catch (IOException ex) {
logger.log(Level.WARNING, "There was an error reading CyberTriage license to file: " + licenseFile.getAbsolutePath(), ex);
logger.log(Level.WARNING, "There was an error reading malware ingest settings from file: " + settingsFile.getAbsolutePath(), ex);
}
}
if (settings == null) {
settings = new MalwareIngestSettings();
}
return toRet;
return settings;
}
private File getCTLicenseFile() {
return Paths.get(PlatformUtil.getModuleConfigDirectory(), CT_SETTINGS_DIR, CT_LICENSE_FILENAME).toFile();
}
......
Core/src/com/basistech/df/cybertriage/autopsy/ctoptions/ctcloud/CTMalwareScannerOptionsPanel.java
+ 17
− 1
View file @ fe26a6e0
......@@ -111,6 +111,7 @@ public void componentShown(ComponentEvent e) {
@Override
public synchronized void saveSettings() {
ctPersistence.saveLicenseResponse(getLicenseInfo());
ctPersistence.saveMalwareSettings(getIngestSettings());
}
@Override
......@@ -127,11 +128,26 @@ public synchronized void loadSettings() {
if (licenseInfo != null) {
loadMalwareScansInfo(licenseInfo);
}
MalwareIngestSettings ingestSettings = ctPersistence.loadMalwareIngestSettings();
setIngestSettings(ingestSettings);
}
private synchronized LicenseResponse getLicenseInfo() {
return this.licenseInfo == null ? null : this.licenseInfo.getLicenseResponse();
}
private MalwareIngestSettings getIngestSettings() {
return new MalwareIngestSettings()
.setUploadFiles(this.fileUploadCheckbox.isSelected());
}
private void setIngestSettings(MalwareIngestSettings ingestSettings) {
if (ingestSettings == null) {
ingestSettings = new MalwareIngestSettings();
}
this.fileUploadCheckbox.setSelected(ingestSettings.isUploadFiles());
}
private synchronized void setLicenseDisplay(LicenseInfo licenseInfo, String licenseMessage) {
this.licenseInfo = licenseInfo;
......@@ -411,7 +427,7 @@ private void licenseInfoAddButtonActionPerformed(java.awt.event.ActionEvent evt)
}//GEN-LAST:event_licenseInfoAddButtonActionPerformed
private void fileUploadCheckboxActionPerformed(java.awt.event.ActionEvent evt) {//GEN-FIRST:event_fileUploadCheckboxActionPerformed
// TODO add your handling code here:
this.firePropertyChange(OptionsPanelController.PROP_CHANGED, null, null);
}//GEN-LAST:event_fileUploadCheckboxActionPerformed
@NbBundle.Messages({
......
Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/MalwareScanIngestModule.java
+ 82
− 49
View file @ fe26a6e0
......@@ -40,6 +40,7 @@
import java.util.stream.Stream;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.curator.shaded.com.google.common.collect.Lists;
import org.openide.util.NbBundle.Messages;
import org.sleuthkit.autopsy.casemodule.Case;
import org.sleuthkit.autopsy.coreutils.Logger;
......@@ -96,6 +97,9 @@ private static class SharedProcessing {
private static final long MIN_UPLOAD_SIZE = 1;
private static final long MAX_UPLOAD_SIZE = 1_000_000_000;
private static final int NUM_FILE_UPLOAD_RETRIES = 60 * 5;
private static final long FILE_UPLOAD_RETRY_SLEEP_MILLIS = 60 * 1000;
private static final Set<String> EXECUTABLE_MIME_TYPES = Stream.of(
"application/x-bat",//NON-NLS
......@@ -128,6 +132,7 @@ private static class SharedProcessing {
private BlackboardArtifact.Type malwareType = null;
private long dsId = 0;
private long ingestJobId = 0;
private boolean uploadUnknownFiles = false;
@Messages({
"MalwareScanIngestModule_ShareProcessing_lowLimitWarning_title=Hash Lookups Low",
......@@ -186,6 +191,7 @@ synchronized void startUp(IngestJobContext context) throws IngestModuleException
dsId = context.getDataSource().getId();
ingestJobId = context.getJobId();
licenseInfo = licenseInfoOpt.get();
uploadUnknownFiles = ctSettingsPersistence.loadMalwareIngestSettings().isUploadFiles();
// set run state to initialized
runState = RunState.STARTED_UP;
......@@ -201,55 +207,6 @@ private static long remaining(Long limit, Long used) {
return limit - used;
}
private boolean isUnknown(CTCloudBean cloudBean) {
return cloudBean != null
&& cloudBean.getMalwareResult() != null
&& cloudBean.getMalwareResult().getStatus() == MalwareResultBean.Status.NOT_FOUND;
}
private boolean isUploadable(AbstractFile af) {
long size = af.getSize();
return size >= MIN_UPLOAD_SIZE && size <= MAX_UPLOAD_SIZE;
}
private boolean uploadFile(SleuthkitCase skCase, DecryptedLicenseResponse decrypted, CTCloudBean cloudBean, long objId) throws CTCloudException, TskCoreException {
if (!isUnknown(cloudBean)) {
return false;
}
AbstractFile af = skCase.getAbstractFileById(objId);
if (af == null) {
return false;
}
if (!isUploadable(af)) {
return false;
}
// get auth token / file upload url
AuthTokenResponse authTokenResponse = ctApiDAO.getAuthToken(decrypted, true);
if (StringUtils.isBlank(authTokenResponse.getFileUploadUrl())) {
throw new CTCloudException(CTCloudException.ErrorCode.NETWORK_ERROR);
}
// upload bytes
ReadContentInputStream fileInputStream = new ReadContentInputStream(af);
ctApiDAO.uploadFile(authTokenResponse.getFileUploadUrl(), af.getName(), fileInputStream);
// upload metadata
MetadataUploadRequest metaRequest = new MetadataUploadRequest()
.setCreatedDate(af.getCrtime())
.setFilePath(af.getUniquePath())
.setFileSizeBytes(af.getSize())
.setFileUploadUrl(authTokenResponse.getFileUploadUrl())
.setMd5(af.getMd5Hash())
.setSha1(af.getSha1Hash())
.setSha256(af.getSha256Hash());
ctApiDAO.uploadMeta(new AuthenticatedRequestData(decrypted, authTokenResponse), metaRequest);
return true;
}
@Messages({
"MalwareScanIngestModule_ShareProcessing_batchTimeout_title=Batch Processing Timeout",
"MalwareScanIngestModule_ShareProcessing_batchTimeout_desc=Batch processing timed out"
......@@ -391,6 +348,82 @@ private String sanitizedMd5(String orig) {
return StringUtils.defaultString(orig).trim().toLowerCase();
}
private boolean isUnknown(CTCloudBean cloudBean) {
return cloudBean != null
&& cloudBean.getMalwareResult() != null
&& cloudBean.getMalwareResult().getStatus() == MalwareResultBean.Status.NOT_FOUND;
}
private boolean isUploadable(AbstractFile af) {
long size = af.getSize();
return size >= MIN_UPLOAD_SIZE && size <= MAX_UPLOAD_SIZE;
}
private boolean uploadFile(CTCloudBean cloudBean, long objId) throws CTCloudException, TskCoreException {
if (!uploadUnknownFiles) {
return false;
}
if (!isUnknown(cloudBean)) {
return false;
}
AbstractFile af = skCase.getAbstractFileById(objId);
if (af == null) {
return false;
}
if (!isUploadable(af)) {
return false;
}
// get auth token / file upload url
AuthTokenResponse authTokenResponse = ctApiDAO.getAuthToken(decrypted, true);
if (StringUtils.isBlank(authTokenResponse.getFileUploadUrl())) {
throw new CTCloudException(CTCloudException.ErrorCode.NETWORK_ERROR);
} else if (remaining(authTokenResponse.getFileUploadLimit(), authTokenResponse.getFileUploadCount()) <= 0) {
// don't proceed with upload if reached limit
uploadUnknownFiles = false;
return false;
}
// upload bytes
ReadContentInputStream fileInputStream = new ReadContentInputStream(af);
ctApiDAO.uploadFile(authTokenResponse.getFileUploadUrl(), af.getName(), fileInputStream);
// upload metadata
MetadataUploadRequest metaRequest = new MetadataUploadRequest()
.setCreatedDate(af.getCrtime())
.setFilePath(af.getUniquePath())
.setFileSizeBytes(af.getSize())
.setFileUploadUrl(authTokenResponse.getFileUploadUrl())
.setMd5(af.getMd5Hash())
.setSha1(af.getSha1Hash())
.setSha256(af.getSha256Hash());
ctApiDAO.uploadMeta(new AuthenticatedRequestData(decrypted, authTokenResponse), metaRequest);
return true;
}
private boolean getUploadedFileResults(Map<String, List<Long>> md5objIdMapping) {
Map<String, List<Long>> remaining = new HashMap<>(md5objIdMapping);
for (int retry = 0; retry < NUM_FILE_UPLOAD_RETRIES; retry++) {
List<List<String>> md5Batches = Lists.partition(new ArrayList<>(remaining.keySet()), BATCH_SIZE);
for (List<String> batch : md5Batches) {
// TODO query and capture still unknown
}
if (remaining.isEmpty()) {
return true;
}
Thread.sleep(FILE_UPLOAD_RETRY_SLEEP_MILLIS);
}
}
@Messages({
"MalwareScanIngestModule_SharedProcessing_createAnalysisResult_Yes=YES",
"MalwareScanIngestModule_SharedProcessing_createAnalysisResult_No=NO"
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment