Merge pull request #2085 from dannysmyda/6993-File-Repo-HTTP

6993 File Repo Native HTTP

bindings/java/ivy.xml

@@ -20,6 +20,9 @@
 		</dependency>
 		<dependency org="com.mchange" name="c3p0" rev="0.9.5" />
 		<dependency org="com.zaxxer" name="SparseBitSet" rev="1.1" />
+		
+		<dependency org="org.apache.httpcomponents" name="httpmime" rev="4.5.13"/>
+		<dependency org="org.apache.httpcomponents" name="httpclient" rev="4.5.13"/>
 	</dependencies>
 </ivy-module>
 

bindings/java/nbproject/project.xml

@@ -114,7 +114,7 @@
         <java-data xmlns="http://www.netbeans.org/ns/freeform-project-java/4">
             <compilation-unit>
                 <package-root>src</package-root>
-                <classpath mode="compile">lib;lib/diffutils-1.2.1.jar;lib/junit-4.8.2.jar;lib/postgresql-9.4-1201.jdbc41.jar;lib/c3p0-0.9.5.jar;lib/mchange-commons-java-0.2.9.jar;lib/c3p0-0.9.5-sources.jar;lib/c3p0-0.9.5-javadoc.jar;lib/joda-time-2.4.jar;lib/commons-lang3-3.0.jar;lib/guava-19.0.jar;lib/SparseBitSet-1.1.jar;lib/gson-2.8.5.jar;lib/commons-validator-1.6.jar</classpath>
+                <classpath mode="compile">lib;lib/diffutils-1.2.1.jar;lib/junit-4.8.2.jar;lib/postgresql-9.4-1201.jdbc41.jar;lib/c3p0-0.9.5.jar;lib/mchange-commons-java-0.2.9.jar;lib/c3p0-0.9.5-sources.jar;lib/c3p0-0.9.5-javadoc.jar;lib/joda-time-2.4.jar;lib/commons-lang3-3.0.jar;lib/guava-19.0.jar;lib/SparseBitSet-1.1.jar;lib/gson-2.8.5.jar;lib/commons-validator-1.6.jar;lib/httpclient-4.5.13.jar;lib/httpcore-4.4.13.jar;lib/commons-logging-1.2.jar;lib/commons-codec-1.11.jar;lib/httpmime-4.5.13.jar</classpath>
                 <built-to>build</built-to>
                 <source-level>1.8</source-level>
             </compilation-unit>

bindings/java/src/org/sleuthkit/datamodel/AbstractFile.java

@@ -20,7 +20,13 @@
 
 import java.io.FileNotFoundException;
 import java.io.IOException;
+import java.io.InputStream;
 import java.io.RandomAccessFile;
+import java.nio.file.FileAlreadyExistsException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.StandardCopyOption;
 import java.sql.SQLException;
 import java.sql.Statement;
 import java.text.MessageFormat;
@@ -1073,13 +1079,16 @@ private synchronized void loadLocalFile() throws TskCoreException {
 				throw new TskCoreException("Previously failed to load file with object ID " + getId() + " from file repository.");
 			}
 			
-			// Copy the file from the server
-			try {
-				localFile = FileRepository.downloadFromFileRepository(this);
-			} catch (TskCoreException ex) {
+			try (InputStream fileRepositoryStream = FileRepository.download(this)) {
+				Path localFilePath = Paths.get(FileRepository.getTempDirectory().getAbsolutePath(), this.getSha256Hash());
+				if (!Files.exists(localFilePath)) {
+					Files.copy(fileRepositoryStream, localFilePath);
+				}
+				localFile = localFilePath.toFile();
+			} catch (FileRepositoryException | IOException ex) {
 				// If we've failed to download from the file repository, don't try again for this session.
 				errorLoadingFromFileRepo = true;
-				throw ex;
+				throw new TskCoreException("Failed trying to download file from repository", ex);
 			}
 		}
 	}

bindings/java/src/org/sleuthkit/datamodel/BulkExistenceEnum.java

+/*
+ * SleuthKit Java Bindings
+ *
+ * Copyright 2020 Basis Technology Corp.
+ * Contact: carrier <at> sleuthkit <dot> org
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.sleuthkit.datamodel;
+
+/**
+ * Represents the states of existence for a file in the file repository.
+ * Given that the bulk endpoint is tolerant of dirty SHA-256 values, INVALID
+ * is an enum status for all malformed SHA-256 values.
+ */
+public enum BulkExistenceEnum {
+	TRUE,
+	FALSE,
+	INVALID;
+}

bindings/java/src/org/sleuthkit/datamodel/BulkExistenceResult.java

+/*
+ * SleuthKit Java Bindings
+ *
+ * Copyright 2020 Basis Technology Corp.
+ * Contact: carrier <at> sleuthkit <dot> org
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.sleuthkit.datamodel;
+
+import java.util.Collections;
+import java.util.Map;
+
+/**
+ * Container for bulk existence results.
+ */
+public class BulkExistenceResult {
+	
+	private Map<String, BulkExistenceEnum> files;
+	
+	/**
+	 * Checks the status of a file in this container. It is assumed that the
+	 * file is contained within response, otherwise a null value is returned.
+	 * 
+	 * @param file File to test
+	 * @return 
+	 */
+	public BulkExistenceEnum getResult(AbstractFile file) {
+		if (file.getSha256Hash() == null || file.getSha256Hash().isEmpty()) {
+			return BulkExistenceEnum.INVALID;
+		}
+		
+		return files.get(file.getSha256Hash());
+	}
+	
+	void setFiles(Map<String, BulkExistenceEnum> files) {
+		this.files = Collections.unmodifiableMap(files);
+	}
+}

bindings/java/src/org/sleuthkit/datamodel/Bundle.properties

@@ -354,6 +354,6 @@ tagsFilter.displayName.text=Must be tagged
 TextFilter.displayName.text=Must include text:
 TypeFilter.displayName.text=Limit event types to
 FileTypesFilter.displayName.text=Limit file types to
-FileRepository.downloadError.title.text=Error downloading from file repository
-FileRepository.downloadError.msg.text=Failed to download file with object ID {0} and SHA-256 hash {1}
-FileRepository.notEnabled.msg.text=File repository is not enabled
+FileRepository.error.title.text=Error from file repository
+FileRepository.downloadError.msg.text=Encountered unexpected error from the file repository. Please see logs for more information.
+FileRepository.notEnabled.msg.text=File repository is not enabled
\ No newline at end of file

bindings/java/src/org/sleuthkit/datamodel/Bundle.properties-MERGED

@@ -354,6 +354,6 @@ tagsFilter.displayName.text=Must be tagged
 TextFilter.displayName.text=Must include text:
 TypeFilter.displayName.text=Limit event types to
 FileTypesFilter.displayName.text=Limit file types to
-FileRepository.downloadError.title.text=Error downloading from file repository
-FileRepository.downloadError.msg.text=Failed to download file with object ID {0} and SHA-256 hash {1}
-FileRepository.notEnabled.msg.text=File repository is not enabled
+FileRepository.error.title.text=Error from file repository
+FileRepository.downloadError.msg.text=Encountered unexpected error from the file repository. Please see logs for more information.
+FileRepository.notEnabled.msg.text=File repository is not enabled
\ No newline at end of file

bindings/java/src/org/sleuthkit/datamodel/FileRepositoryErrorHandler.java

+/*
+ * SleuthKit Java Bindings
+ *
+ * Copyright 2020 Basis Technology Corp.
+ * Contact: carrier <at> sleuthkit <dot> org
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.sleuthkit.datamodel;
+
+/**
+ * Callback class to use for error reporting.
+ */
+public interface FileRepositoryErrorHandler {
+
+	/**
+	 * Handles displaying an error message to the user (if appropriate).
+	 *
+	 * @param title The title for the error display.
+	 * @param error The more detailed error message to display.
+	 */
+	void displayErrorToUser(String title, String error);
+    
+}

bindings/java/src/org/sleuthkit/datamodel/FileRepositoryException.java

+/*
+ * SleuthKit Java Bindings
+ *
+ * Copyright 2020 Basis Technology Corp.
+ * Contact: carrier <at> sleuthkit <dot> org
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.sleuthkit.datamodel;
+
+public class FileRepositoryException extends Exception {
+	
+	private static final long serialVersionUID = 1L;
+	
+	public FileRepositoryException(String msg) {
+		super(msg);
+	}
+}

bindings/java/src/org/sleuthkit/datamodel/FileRepositorySettings.java

+/*
+ * SleuthKit Java Bindings
+ *
+ * Copyright 2020 Basis Technology Corp.
+ * Contact: carrier <at> sleuthkit <dot> org
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.sleuthkit.datamodel;
+
+/**
+ * Utility class to hold the file repository server settings.
+ */
+public class FileRepositorySettings {
+
+	private final String address;
+	private final String port;
+	
+	/**
+	 * Create a FileRepositorySettings instance for the server.
+	 *
+	 * @param address The IP address/hostname of the server.
+	 * @param port    The port.
+	 */
+	public FileRepositorySettings(String address, String port) {
+		this.address = address;
+		this.port = port;
+	}
+	
+	/**
+	 * Fills in an API template with the address and port.
+	 */
+	String createBaseURL(String urlTemplate) {
+		return String.format(urlTemplate, address, port);
+	}
+}