Initial commit.

README.md

+MR-Leo - Android Client
+========================
+
+This repository contains an Android Studio project.
+
+### Building
+GStreamer development libraries for Android need to be installed on the system building the Android APK. 
+For installing those, see https://gstreamer.freedesktop.org/documentation/installing/for-android-development.html  
+
+Make sure that the system variable _GSTREAMER_ROOT_ANDROID_ has been set on the computer. On Ubuntu this can be fixed by adding
+`export GSTREAMER_ROOT_ANDROID="/<path_to_GStreamer_directory>` to the file _~/.profile_ and restarting the session.  
+
+The app has been tested with GStreamer 1.14.4: https://gstreamer.freedesktop.org/data/pkg/android/1.14.4/  
+
+Note that GStreamer 1.14.4 isn't compatible with the Clang compiler included in Android NDK 18b and later, the versions installed by Android Studio. You thus need to manually install NDK 17c, available at https://developer.android.com/ndk/downloads/older_releases, and then configure the Android Studio project to use that version (File➜Project Structure➜SDK Location➜Android NDK Location)  
+
+It contains a slightly modified version of https://github.com/pedroSG94/rtmp-rtsp-stream-client-java, which is used for streaming H.264 video using Android's MediaCodec framworks.
+
+### How to use Android Studio to build and run the app
+1. Enable Developer settings on your Android device. See https://www.greenbot.com/article/2457986/android/how-to-enable-developer-options-on-your-android-phone-or-tablet.html
+2. Enable USB Debugging in the Developer settings menu on your Android device.
+3. Open Android Studio
+4. Select _File➜Open➜Open File or Project_. Select the project.
+5. Connect your phone using a USB cable.
+6. Select _Build➜Make Project_.
+7. After it has successfully built the project, select _Run➜Run 'app'_.
+8. In the new options menu, select your phone.
+
+### Connecting the app to the server
+1. Make sure that the Android device and the PC that will act as a server are on the same WiFi network.
+2. Start the server (MR-Leo-server).
+3. Copy the IP address and port number printed to the console by the server.
+4. Start the app on the Android device.
+5. Enter the above IP address and port number to the main menu interface.
+6. Press the button _Edge MR_.
+7. After a point cloud is visible, press the green button to insert a 3D object to the scene. Remove the object with the red button.
+8. Change the configuration using the main menu (accesible from the menu button in the screen's top right).
+
+### Known limitations
+* Severe stuttering has been observed. It is under investigation.
+* The camera streaming doesn't support resuming operations after it has been suspended, i.e. the user has gone to the Android home screen or to another app. After it has been stopped it has to first be closed using the Android active apps list view (on many implementations swiped to the left) before it can be run again.
+* It will probably have many different issues on different phones.
+
+### Authors
+Johan Lindqvist  
+johli392@student.liu.se  
+johan.lindqvist@gmail.com  
+

app/.gitignore

+.externalNativeBuild/
+gst-build-*/
+src/org/freedesktop/gstreamer/GStreamer.java
+src/org/freedesktop/gstreamer/androidmedia

app/AndroidManifest.xml

+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+    package="se.liu.mrleo">
+
+    <uses-permission android:name="android.permission.CAMERA" />
+    <uses-permission android:name="android.permission.INTERNET" />
+    <uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" />
+
+    <uses-feature android:name="android.hardware.camera" />
+    <uses-feature android:name="android.hardware.camera.autofocus" />
+    <uses-feature android:name="android.hardware.camera2.full" />
+    <uses-feature android:glEsVersion="0x00020000"/>
+    <uses-feature android:name="android.hardware.camera.ar" android:required="true"/>
+
+    <!-- Indicates that app requires ARCore ("AR Required"). Ensures app is only
+     visible in the Google Play Store on devices that support ARCore.
+     For "AR Optional" apps comment out this line. -->
+    <!-- <uses-feature android:name="android.hardware.camera.ar" /> -->
+
+    <application
+        android:allowBackup="true"
+        android:label="@string/app_name"
+        android:theme="@style/FullscreenTheme">
+
+        <activity
+            android:screenOrientation="landscape"
+            android:name="se.liu.mrleo.MainActivity">
+            <intent-filter>
+                <action android:name="android.intent.action.MAIN"/>
+                <category android:name="android.intent.category.LAUNCHER"/>
+            </intent-filter>
+        </activity>
+
+        <activity
+            android:screenOrientation="landscape"
+            android:name="se.liu.mrleo.ARCoreActivity"
+            />
+
+        <activity
+            android:screenOrientation="landscape"
+            android:name="se.liu.mrleo.AboutActivity"
+            />
+
+        <activity
+            android:screenOrientation="landscape"
+            android:name="se.liu.mrleo.EdgeActivity"
+            />
+
+        <meta-data android:name="com.google.ar.core" android:value="required" />
+
+    </application>
+</manifest>

app/assets/models/andy_shadow.obj

+# This file uses centimeters as units for non-parametric coordinates.
+
+g default
+v -0.100000 -0.000000 0.100000
+v 0.100000 -0.000000 0.100000
+v -0.100000 0.000000 -0.100000
+v 0.100000 0.000000 -0.100000
+vt 0.000000 0.000000
+vt 1.000000 0.000000
+vt 0.000000 1.000000
+vt 1.000000 1.000000
+vn 0.000000 1.000000 0.000000
+vn 0.000000 1.000000 0.000000
+vn 0.000000 1.000000 0.000000
+vn 0.000000 1.000000 0.000000
+s off
+g AndyBlobShadow_GEO
+f 4/4/1 3/3/2 1/1/3 2/2/4

app/assets/shaders/object.frag

+/*
+ * Copyright 2017 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+precision mediump float;
+
+uniform sampler2D u_Texture;
+
+uniform vec4 u_LightingParameters;
+uniform vec4 u_MaterialParameters;
+uniform vec4 u_ColorCorrectionParameters;
+
+varying vec3 v_ViewPosition;
+varying vec3 v_ViewNormal;
+varying vec2 v_TexCoord;
+
+void main() {
+    // We support approximate sRGB gamma.
+    const float kGamma = 0.4545454;
+    const float kInverseGamma = 2.2;
+    const float kMiddleGrayGamma = 0.466;
+
+    // Unpack lighting and material parameters for better naming.
+    vec3 viewLightDirection = u_LightingParameters.xyz;
+    vec3 colorShift = u_ColorCorrectionParameters.rgb;
+    float averagePixelIntensity = u_ColorCorrectionParameters.a;
+
+    float materialAmbient = u_MaterialParameters.x;
+    float materialDiffuse = u_MaterialParameters.y;
+    float materialSpecular = u_MaterialParameters.z;
+    float materialSpecularPower = u_MaterialParameters.w;
+
+    // Normalize varying parameters, because they are linearly interpolated in the vertex shader.
+    vec3 viewFragmentDirection = normalize(v_ViewPosition);
+    vec3 viewNormal = normalize(v_ViewNormal);
+
+    // Apply inverse SRGB gamma to the texture before making lighting calculations.
+    // Flip the y-texture coordinate to address the texture from top-left.
+    vec4 objectColor = texture2D(u_Texture, vec2(v_TexCoord.x, 1.0 - v_TexCoord.y));
+    objectColor.rgb = pow(objectColor.rgb, vec3(kInverseGamma));
+
+    // Ambient light is unaffected by the light intensity.
+    float ambient = materialAmbient;
+
+    // Approximate a hemisphere light (not a harsh directional light).
+    float diffuse = materialDiffuse *
+            0.5 * (dot(viewNormal, viewLightDirection) + 1.0);
+
+    // Compute specular light.
+    vec3 reflectedLightDirection = reflect(viewLightDirection, viewNormal);
+    float specularStrength = max(0.0, dot(viewFragmentDirection, reflectedLightDirection));
+    float specular = materialSpecular *
+            pow(specularStrength, materialSpecularPower);
+
+    vec3 color = objectColor.rgb * (ambient + diffuse) + specular;
+    // Apply SRGB gamma before writing the fragment color.
+    color.rgb = pow(color, vec3(kGamma));
+    // Apply average pixel intensity and color shift
+    color *= colorShift * (averagePixelIntensity / kMiddleGrayGamma);
+    gl_FragColor.rgb = color;
+    gl_FragColor.a = objectColor.a;
+}

app/assets/shaders/object.vert

+/*
+ * Copyright 2017 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+uniform mat4 u_ModelView;
+uniform mat4 u_ModelViewProjection;
+
+attribute vec4 a_Position;
+attribute vec3 a_Normal;
+attribute vec2 a_TexCoord;
+
+varying vec3 v_ViewPosition;
+varying vec3 v_ViewNormal;
+varying vec2 v_TexCoord;
+
+void main() {
+    v_ViewPosition = (u_ModelView * a_Position).xyz;
+    v_ViewNormal = normalize((u_ModelView * vec4(a_Normal, 0.0)).xyz);
+    v_TexCoord = a_TexCoord;
+    gl_Position = u_ModelViewProjection * a_Position;
+}

app/assets/shaders/plane.frag

+/*
+ * Copyright 2017 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+precision highp float;
+uniform sampler2D u_Texture;
+uniform vec4 u_dotColor;
+uniform vec4 u_lineColor;
+uniform vec4 u_gridControl;  // dotThreshold, lineThreshold, lineFadeShrink, occlusionShrink
+varying vec3 v_TexCoordAlpha;
+
+void main() {
+  vec4 control = texture2D(u_Texture, v_TexCoordAlpha.xy);
+  float dotScale = v_TexCoordAlpha.z;
+  float lineFade = max(0.0, u_gridControl.z * v_TexCoordAlpha.z - (u_gridControl.z - 1.0));
+  vec3 color = (control.r * dotScale > u_gridControl.x) ? u_dotColor.rgb
+             : (control.g > u_gridControl.y)            ? u_lineColor.rgb * lineFade
+                                                        : (u_lineColor.rgb * 0.25 * lineFade) ;
+  gl_FragColor = vec4(color, v_TexCoordAlpha.z * u_gridControl.w);
+}

app/assets/shaders/plane.vert

+/*
+ * Copyright 2017 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+uniform mat4 u_Model;
+uniform mat4 u_ModelViewProjection;
+uniform mat2 u_PlaneUvMatrix;
+uniform vec3 u_Normal;
+
+attribute vec3 a_XZPositionAlpha; // (x, z, alpha)
+
+varying vec3 v_TexCoordAlpha;
+
+void main() {
+   vec4 local_pos = vec4(a_XZPositionAlpha.x, 0.0, a_XZPositionAlpha.y, 1.0);
+   vec4 world_pos = u_Model * local_pos;
+
+   // Construct two vectors that are orthogonal to the normal.
+   // This arbitrary choice is not co-linear with either horizontal
+   // or vertical plane normals.
+   const vec3 arbitrary = vec3(1.0, 1.0, 0.0);
+   vec3 vec_u = normalize(cross(u_Normal, arbitrary));
+   vec3 vec_v = normalize(cross(u_Normal, vec_u));
+
+   // Project vertices in world frame onto vec_u and vec_v.
+   vec2 uv = vec2(dot(world_pos.xyz, vec_u), dot(world_pos.xyz, vec_v));
+   v_TexCoordAlpha = vec3(u_PlaneUvMatrix * uv, a_XZPositionAlpha.z);
+   gl_Position = u_ModelViewProjection * local_pos;
+}

app/assets/shaders/point_cloud.frag

+/*
+ * Copyright 2017 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+precision mediump float;
+varying vec4 v_Color;
+
+void main() {
+    gl_FragColor = v_Color;
+}

app/assets/shaders/point_cloud.vert

+/*
+ * Copyright 2017 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+uniform mat4 u_ModelViewProjection;
+uniform vec4 u_Color;
+uniform float u_PointSize;
+
+attribute vec4 a_Position;
+
+varying vec4 v_Color;
+
+void main() {
+   v_Color = u_Color;
+   gl_Position = u_ModelViewProjection * vec4(a_Position.xyz, 1.0);
+   gl_PointSize = u_PointSize;
+}

app/assets/shaders/screenquad.frag

+/*
+ * Copyright 2017 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#extension GL_OES_EGL_image_external : require
+
+precision mediump float;
+varying vec2 v_TexCoord;
+uniform samplerExternalOES sTexture;
+
+
+void main() {
+    gl_FragColor = texture2D(sTexture, v_TexCoord);
+}

app/assets/shaders/screenquad.vert

+/*
+ * Copyright 2017 Google Inc. All Rights Reserved.
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+attribute vec4 a_Position;
+attribute vec2 a_TexCoord;
+
+varying vec2 v_TexCoord;
+
+void main() {
+   gl_Position = a_Position;
+   v_TexCoord = a_TexCoord;
+}

app/build.gradle

+apply plugin: 'com.android.application'
+
+android {
+    compileSdkVersion 28
+
+    defaultConfig {
+        applicationId "se.liu.mrleo"
+        minSdkVersion 23
+        versionCode 1
+        versionName "1.0"
+
+        externalNativeBuild {
+            ndkBuild {
+                def gstRoot
+                if (project.hasProperty('gstAndroidRoot'))
+                    gstRoot = project.gstAndroidRoot
+                else
+                    gstRoot = System.env.GSTREAMER_ROOT_ANDROID
+                if (gstRoot == null)
+                    throw new FileNotFoundException('GSTREAMER_ROOT_ANDROID not set')
+
+                arguments "NDK_APPLICATION_MK=jni/Application.mk",
+                        "GSTREAMER_JAVA_SRC_DIR=src",
+                        "GSTREAMER_ROOT_ANDROID=$gstRoot",
+                        "GSTREAMER_ASSETS_DIR=src/main/assets"
+
+                targets "gstmredge"
+
+                abiFilters  'armeabi-v7a', 'arm64-v8a', 'x86'
+            }
+        }
+    }
+
+    buildTypes {
+        release {
+            minifyEnabled false
+            proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.pro'
+            sourceSets {
+                main {
+                    manifest.srcFile 'AndroidManifest.xml'
+                    java.srcDirs = ['src']
+                    resources.srcDirs = ['src']
+                    aidl.srcDirs = ['src']
+                    renderscript.srcDirs = ['src']
+                    res.srcDirs = ['res']
+                    assets.srcDirs = ['assets']
+                }
+            }
+        }
+    }
+
+
+    externalNativeBuild {
+        ndkBuild {
+            path 'jni/Android.mk'
+        }
+    }
+
+    buildToolsVersion = '28.0.3'
+    compileOptions {
+        sourceCompatibility = '1.8'
+        targetCompatibility = '1.8'
+    }
+}
+
+afterEvaluate {
+    if (project.hasProperty('compileDebugJavaWithJavac')) {
+        project.compileDebugJavaWithJavac.dependsOn 'externalNativeBuildDebug'
+    }
+    if (project.hasProperty('compileReleaseJavaWithJavac')) {
+        project.compileReleaseJavaWithJavac.dependsOn 'externalNativeBuildRelease'
+    }
+}
+
+
+dependencies {
+    implementation fileTree(dir: 'libs', include: ['*.jar'])
+
+    api project(':rtplibrary')
+
+    // Edge
+    implementation 'com.obsez.android.lib.filechooser:filechooser:1.1.19'
+    implementation 'com.android.support:appcompat-v7:28.0.0'
+    implementation 'com.android.support:design:28.0.0'
+
+    // ARCore
+    implementation 'de.javagl:obj:0.3.0'
+    implementation 'com.google.ar:core:1.9.0'
+}

app/jni/Android.mk

+LOCAL_PATH := $(call my-dir)
+
+include $(CLEAR_VARS)
+
+LOCAL_MODULE    := gstmredge
+LOCAL_SRC_FILES := gstmredge.c
+
+LOCAL_SHARED_LIBRARIES := gstreamer_android
+LOCAL_LDLIBS := -llog -landroid
+
+include $(BUILD_SHARED_LIBRARY)
+
+ifndef GSTREAMER_ROOT_ANDROID
+$(error GSTREAMER_ROOT_ANDROID is not defined!)
+endif
+
+ifeq ($(TARGET_ARCH_ABI),armeabi)
+GSTREAMER_ROOT := $(GSTREAMER_ROOT_ANDROID)/arm
+else ifeq ($(TARGET_ARCH_ABI),armeabi-v7a)
+GSTREAMER_ROOT := $(GSTREAMER_ROOT_ANDROID)/armv7
+else ifeq ($(TARGET_ARCH_ABI),arm64-v8a)
+GSTREAMER_ROOT := $(GSTREAMER_ROOT_ANDROID)/arm64
+else ifeq ($(TARGET_ARCH_ABI),x86)
+GSTREAMER_ROOT := $(GSTREAMER_ROOT_ANDROID)/x86
+else ifeq ($(TARGET_ARCH_ABI),x86_64)
+GSTREAMER_ROOT := $(GSTREAMER_ROOT_ANDROID)/x86_64
+else
+$(error Target arch ABI not supported: $(TARGET_ARCH_ABI))
+endif
+
+GSTREAMER_NDK_BUILD_PATH  := $(GSTREAMER_ROOT)/share/gst-android/ndk-build/
+
+
+include $(GSTREAMER_NDK_BUILD_PATH)/plugins.mk
+GSTREAMER_PLUGINS := $(GSTREAMER_PLUGINS_CORE) \
+                     $(GSTREAMER_PLUGINS_PLAYBACK) \
+                     $(GSTREAMER_PLUGINS_EFFECTS) \
+                     $(GSTREAMER_PLUGINS_CODECS) \
+                     $(GSTREAMER_PLUGINS_BAD) \
+                     $(GSTREAMER_PLUGINS_NET) \
+                     $(GSTREAMER_PLUGINS_SYS) \
+                     $(GSTREAMER_PLUGINS_CODECS_RESTRICTED)
+
+GSTREAMER_EXTRA_DEPS := gstreamer-video-1.0
+
+include $(GSTREAMER_NDK_BUILD_PATH)/gstreamer-1.0.mk

app/jni/Application.mk

+APP_ABI = armeabi-v7a arm64-v8a x86