Move documentation to DataModel

.gitignore

@@ -5,6 +5,7 @@
 /bindings/java/lib/
 /bindings/java/build/
 /bindings/java/dist
+/bindings/java/doxygen/tskjni_doxygen.tag
 /bindings/java/test/output/results
 /bindings/java/test/output/gold/dummy
 /bindings/java/test/output/gold/*_BU.txt

bindings/java/build.xml

@@ -126,13 +126,13 @@
 		</javac>
 	</target>
 
-	<target name="SQLite" depends="check-build, init-ivy, compile, copyLibsSQLite"
+	<target name="SQLite" depends="doxygen, check-build, init-ivy, compile, copyLibsSQLite"
 			unless="up-to-date" description="generate the distribution" >
 		<!-- Put everything in ${build} into the MyProject-${DSTAMP}.jar file -->
 		<jar jarfile="${dist}/Tsk_DataModel.jar" basedir="${build}"/>
 	</target>
 
-	<target name="PostgreSQL" depends="check-build, init-ivy, compile, copyLibs-PostgreSQL" 
+	<target name="PostgreSQL" depends="doxygen, check-build, init-ivy, compile, copyLibs-PostgreSQL" 
 			unless="up-to-date" description="generate the PostgreSQL distribution" >
 		<!-- Put everything in ${build} into the MyProject-${DSTAMP}.jar file -->
 		<jar jarfile="${dist}/Tsk_DataModel.jar" basedir="${build}"/>
@@ -158,13 +158,13 @@
 		<antcall target="Debug-SQLite"/>
 	 </target>
 	
-	<target name="Debug-SQLite" depends="check-build, init-ivy, compile, copyLibsSQLiteDebug" 
+	<target name="Debug-SQLite" depends="doxygen, check-build, init-ivy, compile, copyLibsSQLiteDebug" 
 			unless="up-to-date" description="generate the debug distribution" >
 		<!-- Put everything in ${build} into the MyProject-${DSTAMP}.jar file -->
 		<jar jarfile="${dist}/Tsk_DataModel.jar" basedir="${build}"/>
 	</target>
   
-	<target name="Debug-PostgreSQL" depends="init-ivy, compile, copyLibsDebug-PostgreSQL" 
+	<target name="Debug-PostgreSQL" depends="doxygen, init-ivy, compile, copyLibsDebug-PostgreSQL" 
 			description="generate the PostgreSQL debug distribution" >
 		<!-- Put everything in ${build} into the MyProject-${DSTAMP}.jar file -->
 		<jar jarfile="${dist}/Tsk_DataModel.jar" basedir="${build}"/>
@@ -209,5 +209,10 @@
 		</java>
 		
 	</target>
+	<target name="doxygen" description="build doxygen docs, requires doxygen in PATH">
+		<exec executable="doxygen" dir="${basedir}/doxygen">
+			<arg value="Doxyfile"/>
+		</exec>
+	</target>
 	
 </project>

bindings/java/doxygen/insert_and_update_database.dox

+/*! \page insert_and_update_database_page INSERT and UPDATE
+
+\section types_of_databases_available Inserting Into the Database
+This page is for people who are developing code for the Autopsy framework and need to place items into the database with SQL statements. If you are not developing Autopsy framework code (Autopsy Modules are not Autopsy framework code), you should have no need to create SQL statements to insert items into the database. If you developing an Autopsy module, please see \ref query_database_page.
+
+Autopsy currently allows either \ref sqlite_db or \ref postgresql_db as the back-end database system for a case. Any code you write could be used with either as the backend database, at the user's discretion. Be sure to test your work with both platforms.
+
+
+\subsection sqlite_db SQLite
+- Use SQL statments supported by SQLite 3
+- Used for Single-user cases
+- Built-in
+- No configuration required
+- Databases are stored locally
+
+\subsection postgresql_db PostgreSQL
+- Use SQL statments supported by PostgreSQL 9.4
+- Used for Multi-user cases
+- An accessible instance of PostgreSQL must be running (on a server, or locally)
+- Proper configuration to connect to this instance of PostgreSQL required
+- Databases are stored within PostgreSQL, wherever it happens to be running
+
+
+<br>
+\section insert_pitfalls_to_avoid How to Avoid Pitfalls When INSERTing into the Database
+- Do not use INSERT OR REPLACE INTO. It does not exist in PostgreSQL.
+<br>
+<br>
+- Do not use INSERT OR IGNORE INTO. It does not exist in PostgreSQL.
+<br>
+<br>
+- Do not insert [NUL characters](http://en.wikipedia.org/wiki/Null_character) into the database as UTF-8 (NUL characters are not NULL fields). Translate NUL characters to the [SUB character](http://en.wikipedia.org/wiki/Substitute_character) with the following instead:
+\code{.java}
+private String replaceNulls(String text);
+\endcode
+<br>
+
+*/
\ No newline at end of file

bindings/java/doxygen/main.dox

@@ -4,7 +4,7 @@
 
 These classes allow Java programs to access data extracted by The Sleuth Kit.
 
-The Sleuth Kit is primarily a C/C++ library and set of command line tools. These classes allow programs to obtain the data that TSK can produce.   The typical steps would be to use JNI to cause the TSK library to create and populate a SQLite database.  The Java classes then directly open the SQLite database and perform queries on it. 
+The Sleuth Kit is primarily a C/C++ library and set of command line tools. These classes allow programs to obtain the data that TSK can produce.   The typical steps would be to use JNI to cause the TSK library to create and populate a SQLite or PostgreSQL database.  The Java classes then directly open the database and perform queries on it. 
 
 \section basics Basic
 
@@ -43,6 +43,11 @@ To find artifacts, you have two general options.  If you have an org.sleuthkit.d
 
 If you want to create an artifact type that is not defined in the framework/bindings, then use org.sleuthkit.datamodel.SleuthkitCase.addArtifactType() to define the type.  This will return back an artifact ID that you can pass in to create actual artifacts.  Note that each database instance could assign a different ID for the custom attributes and artifacts.  It all depends on what modules were run before it and if they added their own types.  Later modules will need the ID of the new type if they want to find artifacts on the blackboard.  They can get the ID for that database using org.sleuthkit.datamodel.SleuthkitCase.getArtifactTypeID().   Similarly, you can add custom attributes with org.sleuthkit.datamodel.SleuthkitCase.addAttrType() and find the ID using org.sleuthkit.datamodel.SleuthkitCase.getAttrTypeID().
 
+\section the_database The Database
+How to \subpage query_database_page
+<br>
+How to \subpage insert_and_update_database_page into the Database
+
 
 */
 

bindings/java/doxygen/query_database.dox

+/*! \page query_database_page Query the Database
+\section types_of_databases Database Queries
+This page is for people who are developing their own Autopsy plugin modules that require SQL queries. If you are not developing a module requiring SQL queries, you can skip this page.
+Autopsy currently allows either SQLite or PostgreSQL as the back-end database system for a case. Any module you write could be used with either as the backend database, at the user's discretion.
+
+If you are writing code actually for Autopsy, not just an Autopsy module, you may need to be able to INSERT and UPDATE into the database as well. Please see \subpage insert_and_update_database_page.
+
+<br>
+\subsection which_db Which Database is my Module Accessing?
+In an Autopsy Module, you can check the database type currently in use with the following code snippet:
+\code{.java}
+Case currentCase = Case.getCurrentCase();
+if (currentCase.getCaseType() == Case.CaseType.MULTI_USER_CASE) 
+{
+    // PostgreSQL in use
+}
+else
+{
+    // SQLite in use
+}
+\endcode
+
+
+<br>
+\section db_user_calls TSK methods to Query the Database With User-Supplied SQL
+The following SleuthkitCase methods are available for the user to supply all of, or a portion of, a SQL query. 
+\code{.java}
+ArrayList<BlackboardAttribute> getMatchingAttributes(String whereClause)
+
+ArrayList<BlackboardArtifact> getMatchingArtifacts(String whereClause)
+
+long countFilesWhere(String sqlWhereClause)
+
+List<AbstractFile> findAllFilesWhere(String sqlWhereClause)
+
+List<Long> findAllFileIdsWhere(String sqlWhereClause)
+
+CaseDbQuery executeQuery(String query)
+
+List<FsContent> findFilesWhere(String sqlWhereClause) [deprecated]
+
+ResultSet runQuery(String query) [deprecated]
+
+void closeRunQuery(ResultSet resultSet) [deprecated]
+\endcode
+
+The majority of them only allow the user to specify a WHERE clause, determining which records to SELECT.
+<br>
+<br>
+The following example finds all the .txt files in the case:
+\code{.java}
+List<AbstractFile> files = sk.findAllFilesWhere("LOWER(name) LIKE '%.txt'");
+\endcode
+
+<br>
+\section db_pitfalls_to_avoid How to Avoid Pitfalls When Using the Query Methods
+Because there are multiple backend databases, care must be taken to use strict SQL. When things must be different between database types, use \ref which_db to determine which database type is currently in use and create the proper SQL statements. Be sure to test your module with both types of databases. They behave differently and will give you different resulting output order.
+<br>
+<br>
+\subsection general_items WHERE Clause Syntax
+- Do not use backticks. PostgreSQL does not use them like SQLite does.
+<br>
+<br>
+- Use only single quotes to quote values. Do not use double quotes for this. Quoting values is not required. 
+<br>
+\code{.java}
+SELECT * FROM tsk_files WHERE has_path = "1"    // Bad example
+SELECT * FROM tsk_files WHERE has_path = '1'    // Good example
+SELECT * FROM tsk_files WHERE has_path = 1      // Good example
+\endcode
+<br>
+- Use only double quotes to quote column names. Do not use single quotes for this. Quoting column names is not required.
+<br>
+\code{.java}
+SELECT 'obj_id' FROM tsk_files WHERE has_path = 1  // Bad example
+SELECT "obj_id" FROM tsk_files WHERE has_path = 1  // Good example
+SELECT obj_id FROM tsk_files WHERE has_path = 1    // Good example
+\endcode
+<br>
+- Do not use || and && to connect logical clauses. This does not exist in PostgreSQL. Use OR and AND instead. 
+\code{.java}
+SELECT COUNT(*) FROM tsk_files WHERE dir_type = '5' &&  md5 IS NULL || size > '0'    // Bad Example
+SELECT COUNT(*) FROM tsk_files WHERE dir_type = '5' AND md5 IS NULL OR size > '0'    // Good Example
+\endcode
+<br>
+- PostgreSQL compares are case-sensitive. Always specify what type of compare you want. UPPER() and LOWER() can help with that.
+\code{.java}
+SELECT * from people WHERE first_name LIKE '%somename%'                 // Will be case sensitive in PostgreSQL, not in SQLite
+SELECT * from people WHERE first_name ILIKE '%somename%'                // Works in PostgreSQL, does not exist in SQLite
+SELECT * from people WHERE LOWER(first_name) LIKE LOWER('%somename%')   // Not case sensitive in either database
+\endcode
+<br>
+- When generating WHERE queries via code, some folks include an AND(1) or OR(0) clause in the query as a placeholder that does not effect the outcome of the query but simplifies the query-generation logic. PostgreSQL does not allow true or false comparisons with integers. The PostgreSql syntax is AND(true) or OR(false). SQLite does not allow the PostgreSQL syntax and PostgreSQL does not allow the SQLite syntax. Do not use this trick to generate queries. Instead, have your code handle the edge cases of if there are no entries for the AND or OR portion of a clause.
+\code{.java}
+WHERE id=12 AND(1)                // SQLite example, will not work in PostgreSQL
+WHERE id=12 AND(true)             // PostgreSQL example, will not work in SQLite
+WHERE id=12                       // Will work in both, just a bit harder to handle all the cases in query-generation code
+\endcode
+<br>
+\subsection order_by How to ORDER BY Consistently
+- SQLite and PostgreSQL have different default sort orders for returned records, so you want to fully specify ORDER BY clauses for both database types. Example:
+\code{.java}
+Case currentCase = Case.getCurrentCase();
+String orderByClause;
+if (currentCase.getCaseType() == Case.CaseType.MULTI_USER_CASE) 
+{
+    orderByClause = "ORDER BY att.value_text, ASC NULLS FIRST"; //PostgreSQL
+}
+else
+{
+    orderByClause = "ORDER BY att.value_text ASC"; //SQLite
+}
+\endcode
+<br>
+- Do not use COLLATE NOCASE to order output. This does not exist in PostgreSQL. Use LOWER() or UPPER() instead.
+\code{.java}
+ORDER BY tsk_files.dir_type, tsk_files.name COLLATE NOCASE  // Bad Example
+ORDER BY tsk_files.dir_type, LOWER(tsk_files.name)          // Good Example
+\endcode
+<br>
+- In ORDER BY clauses, PostgreSQL ignores leading dashes. Given the following data, you will see the following two sort orders for the different databases.
+<br>
+ |   Data   |   PostgreSQL sort order  |   SQLite sort order|
+ |:--------:|:------------------------:|:------------------:|    
+ |Alpha     |        Alpha             |     -Bravo         |
+ |-Bravo    |        -Bravo            |     Alpha          |
+ |Charlie   |        Charlie           |     Charlie        |
+<br>
+To force PostgreSQL to not ignore leading dashes, convert strings to SQL_ASCII before sorting by them. This is done with convert_to(), but it only exists in PostgreSQL. 
+<br>
+\code{.java}
+ORDER BY some_value                            // Bad example
+ORDER BY convert_to(some_value, 'SQL_ASCII')   // Good example
+\endcode
+<br>
+With the code above, using SQL_ASCII encoding, the following results are seen:
+ <br>
+ |   Data   |   PostgreSQL sort order  |   SQLite sort order|
+ |:--------:|:------------------------:|:------------------:|    
+ |Alpha     |        -Bravo            |     -Bravo         |
+ |-Bravo    |        Alpha             |     Alpha          |
+ |Charlie   |        Charlie           |     Charlie        |
+<br>
+- PostgreSQL sorts NULLs last for ASC and first for DESC. SQLite does the opposite. PostgreSQL allows you to control the NULL sort order with NULLS FIRST or NULLS LAST
+\code{.java}
+ORDER BY att.value_text ASC                                        // SQLite example, will give different ordering in PostgreSQL
+ORDER BY convert_to(att.value_text, 'SQL_ASCII') ASC NULLS FIRST   // PostgreSQL example. The command NULLS FIRST does not exist in SQLite, but SQLite will sort nulls first by default.
+\endcode
+<br>
+*/

bindings/java/src/org/sleuthkit/datamodel/SleuthkitCase.java

@@ -1776,6 +1776,7 @@ public ArrayList<BlackboardAttribute> getBlackboardAttributes(final BlackboardAr
 	 * @return a list of matching attributes
 	 * @throws TskCoreException exception thrown if a critical error occurs
 	 * within tsk core
+	 * \ref query_database_page
 	 */
 	public ArrayList<BlackboardAttribute> getMatchingAttributes(String whereClause) throws TskCoreException {
 		CaseDbConnection connection = connections.getConnection();
@@ -1811,6 +1812,7 @@ public ArrayList<BlackboardAttribute> getMatchingAttributes(String whereClause)
 	 * @return a list of matching artifacts
 	 * @throws TskCoreException exception thrown if a critical error occurs
 	 * within tsk core
+	 * \ref query_database_page
 	 */
 	public ArrayList<BlackboardArtifact> getMatchingArtifacts(String whereClause) throws TskCoreException {
 		CaseDbConnection connection = connections.getConnection();
@@ -3334,6 +3336,7 @@ public List<AbstractFile> findFiles(Content dataSource, String fileName, Abstrac
 	 * files (do not begin the WHERE clause with the word WHERE!)
 	 * @return count of files each of which satisfy the given WHERE clause
 	 * @throws TskCoreException
+	 * \ref query_database_page
 	 */
 	public long countFilesWhere(String sqlWhereClause) throws TskCoreException {
 		CaseDbConnection connection = connections.getConnection();
@@ -3363,6 +3366,7 @@ public long countFilesWhere(String sqlWhereClause) throws TskCoreException {
 	 * @return a list of AbstractFile each of which satisfy the given WHERE
 	 * clause
 	 * @throws TskCoreException
+	 * \ref query_database_page
 	 */
 	public List<AbstractFile> findAllFilesWhere(String sqlWhereClause) throws TskCoreException {
 		CaseDbConnection connection = connections.getConnection();
@@ -3390,6 +3394,7 @@ public List<AbstractFile> findAllFilesWhere(String sqlWhereClause) throws TskCor
 	 * files (do not begin the WHERE clause with the word WHERE!)
 	 * @return a list of file ids each of which satisfy the given WHERE clause
 	 * @throws TskCoreException
+	 * \ref query_database_page
 	 */
 	public List<Long> findAllFileIdsWhere(String sqlWhereClause) throws TskCoreException {
 		CaseDbConnection connection = connections.getConnection();
@@ -3424,6 +3429,7 @@ public List<Long> findAllFileIdsWhere(String sqlWhereClause) throws TskCoreExcep
 	 * @deprecated This method is deprecated. Continuing to use this method 
 	 * risks your module not functioning correctly in the future. Use 
 	 * findAllFilesWhere(String sqlWhereClause) instead. 
+	 * \ref query_database_page
 	 */
 	@Deprecated	// use findAllFilesWhere() instead
 	public List<FsContent> findFilesWhere(String sqlWhereClause) throws TskCoreException {
@@ -4184,6 +4190,7 @@ private List<FsContent> resultSetToFsContents(ResultSet rs) throws SQLException
 	 * data from the resultSet
 	 * @throws SQLException if error occurred during the query
 	 * @deprecated Do not use runQuery(), use executeQuery() instead.
+	 * \ref query_database_page
 	 */
 	@Deprecated // Use executeQuery() instead.
 	public ResultSet runQuery(String query) throws SQLException {
@@ -4210,6 +4217,7 @@ public ResultSet runQuery(String query) throws SQLException {
 	 * @throws SQLException of closing the query results failed
 	 * @deprecated Do not use runQuery() and closeRunQuery(), use executeQuery() 
 	 * instead.
+	 * \ref query_database_page
 	 */
 	@Deprecated // Use executeQuery() instead.
 	public void closeRunQuery(ResultSet resultSet) throws SQLException {
@@ -5416,6 +5424,14 @@ ResultSet executeQuery(Statement statement, String query) throws SQLException {
 			return resultSet;
 		}
 
+		/**
+		 * 
+		 * @param statement The SQL statement to execute
+		 * @return returns the ResultSet from the execution of the query
+		 * @throws SQLException 
+		 * \ref query_database_page
+		 * \ref insert_and_update_database_page
+		 */
 		ResultSet executeQuery(PreparedStatement statement) throws SQLException {
 			ResultSet resultSet = null;
 			boolean locked = true;