Merge pull request #7837 from gdicristofaro/AUT-2459-fileUpload

AUT-2459 file upload changes

Core/src/com/basistech/df/cybertriage/autopsy/ctapi/CTApiDAO.java

@@ -25,6 +25,7 @@
 import com.basistech.df.cybertriage.autopsy.ctapi.json.CTCloudBeanResponse;
 import com.basistech.df.cybertriage.autopsy.ctapi.json.DecryptedLicenseResponse;
 import com.basistech.df.cybertriage.autopsy.ctapi.json.FileReputationRequest;
+import com.basistech.df.cybertriage.autopsy.ctapi.json.FileUploadRequest;
 import com.basistech.df.cybertriage.autopsy.ctapi.json.LicenseRequest;
 import com.basistech.df.cybertriage.autopsy.ctapi.json.LicenseResponse;
 import com.basistech.df.cybertriage.autopsy.ctapi.json.MetadataUploadRequest;
@@ -78,21 +79,22 @@ public LicenseResponse getLicenseInfo(String licenseString) throws CTCloudExcept
     }
 
     public AuthTokenResponse getAuthToken(DecryptedLicenseResponse decrypted) throws CTCloudException {
-        return getAuthToken(decrypted, false);
+        return getAuthToken(decrypted, null);
     }
 
-    public AuthTokenResponse getAuthToken(DecryptedLicenseResponse decrypted, boolean fileUpload) throws CTCloudException {
+    public AuthTokenResponse getAuthToken(DecryptedLicenseResponse decrypted, Long fileUploadSize) throws CTCloudException {
         AuthTokenRequest authTokenRequest = new AuthTokenRequest()
                 .setAutopsyVersion(getAppVersion())
-                .setRequestFileUpload(fileUpload)
+                .setRequestFileUpload(fileUploadSize != null && fileUploadSize > 0)
+                .setFileUploadSize(fileUploadSize != null && fileUploadSize > 0 ? fileUploadSize : null)
                 .setBoostLicenseId(decrypted.getBoostLicenseId())
                 .setHostId(decrypted.getLicenseHostId());
 
         return httpClient.doPost(AUTH_TOKEN_REQUEST_PATH, authTokenRequest, AuthTokenResponse.class);
     }
 
-    public void uploadFile(String url, String fileName, InputStream fileIs) throws CTCloudException {
-        httpClient.doFileUploadPost(url, fileName, fileIs);
+    public void uploadFile(FileUploadRequest fileUploadRequest) throws CTCloudException {
+        httpClient.doFileUploadPut(fileUploadRequest);
     }
     
     public void uploadMeta(AuthenticatedRequestData authenticatedRequestData, MetadataUploadRequest metaRequest) throws CTCloudException {

Core/src/com/basistech/df/cybertriage/autopsy/ctapi/CTCloudHttpClient.java

@@ -18,6 +18,8 @@
  */
 package com.basistech.df.cybertriage.autopsy.ctapi;
 
+import com.basistech.df.cybertriage.autopsy.ctapi.CTCloudException.ErrorCode;
+import com.basistech.df.cybertriage.autopsy.ctapi.json.FileUploadRequest;
 import com.basistech.df.cybertriage.autopsy.ctapi.util.ObjectMapperUtil;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import java.io.IOException;
@@ -55,13 +57,14 @@
 import org.apache.http.client.config.RequestConfig;
 import org.apache.http.client.methods.CloseableHttpResponse;
 import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.methods.HttpPut;
 import org.apache.http.client.methods.HttpRequestBase;
 import org.apache.http.impl.client.CloseableHttpClient;
 import org.apache.http.util.EntityUtils;
 import org.apache.http.client.utils.URIBuilder;
 import org.apache.http.entity.ContentType;
+import org.apache.http.entity.InputStreamEntity;
 import org.apache.http.entity.StringEntity;
-import org.apache.http.entity.mime.MultipartEntityBuilder;
 import org.sleuthkit.autopsy.coreutils.Logger;
 import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.http.impl.client.HttpClients;
@@ -184,10 +187,23 @@ public <O> O doPost(String urlPath, Map<String, String> urlReqParams, Object jso
         return null;
     }
 
-    public void doFileUploadPost(String fullUrlPath, String fileName, InputStream fileIs) throws CTCloudException {
-        URI postUri;
+    public void doFileUploadPut(FileUploadRequest fileUploadRequest) throws CTCloudException {
+        if (fileUploadRequest == null) {
+            throw new CTCloudException(ErrorCode.BAD_REQUEST, new IllegalArgumentException("fileUploadRequest cannot be null"));
+        }
+                
+        String fullUrlPath = fileUploadRequest.getFullUrlPath();
+        String fileName = fileUploadRequest.getFileName();
+        InputStream fileInputStream = fileUploadRequest.getFileInputStream();
+        Long contentLength = fileUploadRequest.getContentLength();
+
+        if (StringUtils.isBlank(fullUrlPath) || fileInputStream == null || contentLength == null || contentLength <= 0) {
+            throw new CTCloudException(ErrorCode.BAD_REQUEST, new IllegalArgumentException("fullUrlPath, fileInputStream, contentLength must not be empty, null or less than 0"));
+        }
+        
+        URI putUri;
         try {
-            postUri = new URI(fullUrlPath);
+            putUri = new URI(fullUrlPath);
         } catch (URISyntaxException ex) {
             LOGGER.log(Level.WARNING, "Wrong URL syntax for CT Cloud " + fullUrlPath, ex);
             throw new CTCloudException(CTCloudException.ErrorCode.UNKNOWN, ex);
@@ -195,23 +211,13 @@ public void doFileUploadPost(String fullUrlPath, String fileName, InputStream fi
 
         try (CloseableHttpClient httpclient = createConnection(proxySelector, sslContext)) {
             LOGGER.log(Level.INFO, "initiating http post request to ctcloud server " + fullUrlPath);
-            HttpPost post = new HttpPost(postUri);
-            configureRequestTimeout(post);
-
-            post.addHeader("Connection", "keep-alive");
-
-            MultipartEntityBuilder builder = MultipartEntityBuilder.create();
-            builder.addBinaryBody(
-                    "file",
-                    fileIs,
-                    ContentType.APPLICATION_OCTET_STREAM,
-                    fileName
-            );
+            HttpPut put = new HttpPut(putUri);
+            configureRequestTimeout(put);
 
-            HttpEntity multipart = builder.build();
-            post.setEntity(multipart);
+            put.addHeader("Connection", "keep-alive");
+            put.setEntity(new InputStreamEntity(fileInputStream, contentLength, ContentType.APPLICATION_OCTET_STREAM));
 
-            try (CloseableHttpResponse response = httpclient.execute(post)) {
+            try (CloseableHttpResponse response = httpclient.execute(put)) {
                 int statusCode = response.getStatusLine().getStatusCode();
                 if (statusCode == HttpStatus.SC_OK || statusCode == HttpStatus.SC_NO_CONTENT) {
                     LOGGER.log(Level.INFO, "Response Received. - Status OK");
@@ -381,7 +387,7 @@ private static CloseableHttpClient createConnection(ProxySelector proxySelector,
         HttpClientBuilder builder;
 
         if (ProxySettings.getProxyType() != ProxySettings.DIRECT_CONNECTION
-                && StringUtils.isBlank(ProxySettings.getAuthenticationUsername()) 
+                && StringUtils.isBlank(ProxySettings.getAuthenticationUsername())
                 && ArrayUtils.isEmpty(ProxySettings.getAuthenticationPassword())
                 && WinHttpClients.isWinAuthAvailable()) {
 

Core/src/com/basistech/df/cybertriage/autopsy/ctapi/json/AuthTokenRequest.java

@@ -34,6 +34,9 @@ public class AuthTokenRequest {
     @JsonProperty("requestFileUpload")
     private boolean requestFileUpload;
 
+    @JsonProperty("fileUploadSize")
+    private Long fileUploadSize;
+
     @JsonProperty("host_id")
     private String hostId;
 
@@ -64,6 +67,16 @@ public AuthTokenRequest setRequestFileUpload(boolean requestFileUpload) {
         return this;
     }
 
+    public Long getFileUploadSize() {
+        return fileUploadSize;
+    }
+
+    public AuthTokenRequest setFileUploadSize(Long fileUploadSize) {
+        this.fileUploadSize = fileUploadSize;
+        return this;
+    }
+
+    
     public String getHostId() {
         return hostId;
     }

Core/src/com/basistech/df/cybertriage/autopsy/ctapi/json/FileUploadRequest.java

+/*
+ * Autopsy Forensic Browser
+ *
+ * Copyright 2023 Basis Technology Corp.
+ * Contact: carrier <at> sleuthkit <dot> org
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.basistech.df.cybertriage.autopsy.ctapi.json;
+
+import java.io.InputStream;
+
+/**
+ * Data for a file upload request.
+ */
+public class FileUploadRequest {
+
+    private String fullUrlPath;
+    private String fileName;
+    private InputStream fileInputStream;
+    private Long contentLength;
+
+    public String getFullUrlPath() {
+        return fullUrlPath;
+    }
+
+    public FileUploadRequest setFullUrlPath(String fullUrlPath) {
+        this.fullUrlPath = fullUrlPath;
+        return this;
+    }
+
+    public String getFileName() {
+        return fileName;
+    }
+
+    public FileUploadRequest setFileName(String fileName) {
+        this.fileName = fileName;
+        return this;
+    }
+
+    public InputStream getFileInputStream() {
+        return fileInputStream;
+    }
+
+    public FileUploadRequest setFileInputStream(InputStream fileInputStream) {
+        this.fileInputStream = fileInputStream;
+        return this;
+    }
+
+    public Long getContentLength() {
+        return contentLength;
+    }
+
+    public FileUploadRequest setContentLength(Long contentLength) {
+        this.contentLength = contentLength;
+        return this;
+    }
+
+}

Core/src/com/basistech/df/cybertriage/autopsy/malwarescan/MalwareScanIngestModule.java

@@ -23,6 +23,7 @@
 import com.basistech.df.cybertriage.autopsy.ctapi.json.AuthTokenResponse;
 import com.basistech.df.cybertriage.autopsy.ctapi.json.AuthenticatedRequestData;
 import com.basistech.df.cybertriage.autopsy.ctapi.json.CTCloudBean;
+import com.basistech.df.cybertriage.autopsy.ctapi.json.FileUploadRequest;
 import com.basistech.df.cybertriage.autopsy.ctapi.json.LicenseInfo;
 import com.basistech.df.cybertriage.autopsy.ctapi.json.MalwareResultBean.Status;
 import com.basistech.df.cybertriage.autopsy.ctapi.json.MetadataUploadRequest;
@@ -112,8 +113,10 @@ private static class SharedProcessing {
         //minimum file uploads left before issuing warning
         private static final long LOW_UPLOADS_REMAINING = 25;
 
+        // min and max upload size in bytes
         private static final long MIN_UPLOAD_SIZE = 1;
-        private static final long MAX_UPLOAD_SIZE = 1_000_000_000;
+        private static final long MAX_UPLOAD_SIZE = 100_000_000; // 100MB
+
         private static final int NUM_FILE_UPLOAD_RETRIES = 7;
         private static final long FILE_UPLOAD_RETRY_SLEEP_MILLIS = 60 * 1000;
 
@@ -640,7 +643,7 @@ private boolean uploadFile(IngestJobState ingestJobState, String md5, long objId
             }
 
             // get auth token / file upload url
-            AuthTokenResponse authTokenResponse = ctApiDAO.getAuthToken(ingestJobState.getLicenseInfo().getDecryptedLicense(), true);
+            AuthTokenResponse authTokenResponse = ctApiDAO.getAuthToken(ingestJobState.getLicenseInfo().getDecryptedLicense(), af.getSize());
             if (StringUtils.isBlank(authTokenResponse.getFileUploadUrl())) {
                 throw new CTCloudException(CTCloudException.ErrorCode.NETWORK_ERROR);
             } else if (remaining(authTokenResponse.getFileUploadLimit(), authTokenResponse.getFileUploadCount()) <= 0) {
@@ -658,7 +661,13 @@ private boolean uploadFile(IngestJobState ingestJobState, String md5, long objId
 
             // upload bytes
             ReadContentInputStream fileInputStream = new ReadContentInputStream(af);
-            ctApiDAO.uploadFile(authTokenResponse.getFileUploadUrl(), af.getName(), fileInputStream);
+
+            ctApiDAO.uploadFile(new FileUploadRequest()
+                    .setContentLength(af.getSize())
+                    .setFileInputStream(fileInputStream)
+                    .setFileName(af.getName())
+                    .setFullUrlPath(authTokenResponse.getFileUploadUrl())
+            );
 
             // upload metadata
             MetadataUploadRequest metaRequest = new MetadataUploadRequest()