diff --git a/app/build.gradle.kts b/app/build.gradle.kts index e4f1fc8..233ac14 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -11,9 +11,8 @@ android { applicationId = "com.modarb.android" minSdk = 22 targetSdk = 34 - versionCode = 3 - versionName = "1.2" - + versionCode = 4 + versionName = "2.0" testInstrumentationRunner = "androidx.test.runner.AndroidJUnitRunner" } @@ -71,6 +70,18 @@ dependencies { //Circle progress bar implementation("com.mikhaellopez:circularprogressbar:3.1.0") implementation("androidx.webkit:webkit:1.11.0") + // Pose correction + implementation("com.google.mlkit:pose-detection:18.0.0-beta4") + implementation("com.google.mlkit:pose-detection-accurate:18.0.0-beta4") + // CameraX + implementation("androidx.camera:camera-camera2:1.3.4") + implementation("androidx.camera:camera-lifecycle:1.3.4") + implementation("androidx.camera:camera-view:1.3.4") + + // On Device Machine Learnings + implementation("com.google.guava:guava:32.1.2-jre") + implementation("androidx.camera:camera-core:1.3.4") + implementation("com.google.android.gms:play-services-vision-common:19.1.3") testImplementation("junit:junit:4.13.2") androidTestImplementation("androidx.test.ext:junit:1.1.5") @@ -79,3 +90,9 @@ dependencies { } + +//configurations { +// // Resolves dependency conflict caused by some dependencies use +// // com.google.guava:guava and com.google.guava:listenablefuture together. +// all*.exclude group: 'com.google.guava', module: 'listenablefuture' +//} diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index b012191..cd6e355 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -2,7 +2,14 @@ + + + + + - + + + + + handleRequest( response: Response, onSuccess: (T) -> Unit, onError: (T?) -> Unit ) { diff --git a/app/src/main/java/com/modarb/android/posedetection/CameraActivity.kt b/app/src/main/java/com/modarb/android/posedetection/CameraActivity.kt new file mode 100644 index 0000000..c06ba5b --- /dev/null +++ b/app/src/main/java/com/modarb/android/posedetection/CameraActivity.kt @@ -0,0 +1,170 @@ +package com.modarb.android.posedetection + +import android.content.Intent +import android.os.Bundle +import android.util.Log +import android.widget.CompoundButton +import android.widget.ImageView +import android.widget.Toast +import android.widget.ToggleButton +import androidx.appcompat.app.AppCompatActivity +import com.google.android.gms.common.annotation.KeepName +import com.modarb.android.R +import com.modarb.android.posedetection.Utils.CameraSource +import com.modarb.android.posedetection.Utils.CameraSourcePreview +import com.modarb.android.posedetection.Utils.PreferenceUtils +import com.modarb.android.posedetection.posedetector.PoseDetectorProcessor +import java.io.IOException + +@KeepName +class CameraActivity : AppCompatActivity(), + CompoundButton.OnCheckedChangeListener { + + private var cameraSource: CameraSource? = null + private var preview: CameraSourcePreview? = null + private var graphicOverlay: GraphicOverlay? = null + + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + Log.d(TAG, "onCreate") + setContentView(R.layout.activity_camera_view) + + preview = findViewById(R.id.preview_view) + if (preview == null) { + Log.d(TAG, "Preview is null") + } + + graphicOverlay = findViewById(R.id.graphic_overlay) + if (graphicOverlay == null) { + Log.d(TAG, "graphicOverlay is null") + } + initSetting() + createCameraSource(POSE_DETECTION) + handleCameraSwitch() + } + + private fun initSetting() { + val settingsButton = findViewById(R.id.settings_button) + settingsButton.setOnClickListener { + val intent = Intent(applicationContext, CameraSettingsActivity::class.java) + intent.putExtra( + CameraSettingsActivity.EXTRA_LAUNCH_SOURCE, + CameraSettingsActivity.LaunchSource.LIVE_PREVIEW + ) + startActivity(intent) + } + } + + private fun handleCameraSwitch() { + val facingSwitch = findViewById(R.id.facing_switch) + facingSwitch.setOnCheckedChangeListener(this) + } + + + override fun onCheckedChanged(buttonView: CompoundButton, isChecked: Boolean) { + Log.d(TAG, "Set facing") + if (cameraSource != null) { + if (isChecked) { + cameraSource?.setFacing(CameraSource.CAMERA_FACING_FRONT) + } else { + cameraSource?.setFacing(CameraSource.CAMERA_FACING_BACK) + } + } + preview?.stop() + startCameraSource() + } + + private fun createCameraSource(model: String) { + if (cameraSource == null) { + cameraSource = CameraSource(this, graphicOverlay) + } + try { + when (model) { + + POSE_DETECTION -> { + val poseDetectorOptions = + PreferenceUtils.getPoseDetectorOptionsForLivePreview(this) + Log.i(TAG, "Using Pose Detector with options $poseDetectorOptions") + val shouldShowInFrameLikelihood = + PreferenceUtils.shouldShowPoseDetectionInFrameLikelihoodLivePreview(this) + val visualizeZ = PreferenceUtils.shouldPoseDetectionVisualizeZ(this) + val rescaleZ = PreferenceUtils.shouldPoseDetectionRescaleZForVisualization(this) + val runClassification = + true /*PreferenceUtils.shouldPoseDetectionRunClassification(this)*/ + cameraSource!!.setMachineLearningFrameProcessor( + PoseDetectorProcessor( + this, + poseDetectorOptions, + shouldShowInFrameLikelihood, + visualizeZ, + rescaleZ, + runClassification, + true + ) + ) + } + + else -> Log.e(TAG, "Unknown model: $model") + } + } catch (e: Exception) { + Log.e(TAG, "Can not create image processor: $model", e) + Toast.makeText( + applicationContext, + "Can not create image processor: " + e.message, + Toast.LENGTH_LONG + ).show() + } + } + + private fun startCameraSource() { + if (cameraSource != null) { + try { + if (preview == null) { + Log.d(TAG, "resume: Preview is null") + } + if (graphicOverlay == null) { + Log.d(TAG, "resume: graphOverlay is null") + } + preview!!.start(cameraSource, graphicOverlay) + } catch (e: IOException) { + Log.e(TAG, "Unable to start camera source.", e) + cameraSource!!.release() + cameraSource = null + } + } + } + + public override fun onResume() { + super.onResume() + Log.d(TAG, "onResume") + createCameraSource(POSE_DETECTION) + startCameraSource() + } + + override fun onPause() { + super.onPause() + preview?.stop() + // TODO check that + cameraSource?.release() + } + + override fun onStop() { + super.onStop() + cameraSource?.release() + preview?.stop() + + } + + public override fun onDestroy() { + super.onDestroy() + if (cameraSource != null) { + cameraSource?.release() + } + } + + + companion object { + private const val POSE_DETECTION = "Pose Detection" + private const val TAG = "LivePreviewActivity" + } +} diff --git a/app/src/main/java/com/modarb/android/posedetection/CameraPreferenceFragment.java b/app/src/main/java/com/modarb/android/posedetection/CameraPreferenceFragment.java new file mode 100644 index 0000000..3453e3f --- /dev/null +++ b/app/src/main/java/com/modarb/android/posedetection/CameraPreferenceFragment.java @@ -0,0 +1,109 @@ + + +package com.modarb.android.posedetection; + +import android.hardware.Camera; +import android.os.Bundle; +import android.preference.ListPreference; +import android.preference.PreferenceCategory; +import android.preference.PreferenceFragment; + +import androidx.annotation.StringRes; + +import com.modarb.android.R; +import com.modarb.android.posedetection.Utils.CameraSource; +import com.modarb.android.posedetection.Utils.PreferenceUtils; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class CameraPreferenceFragment extends PreferenceFragment { + + @Override + public void onCreate(Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + + addPreferencesFromResource(R.xml.pref_camera_view); + setUpCameraPreferences(); + } + + void setUpCameraPreferences() { + PreferenceCategory cameraPreference = + (PreferenceCategory) findPreference(getString(R.string.pref_category_key_camera)); + cameraPreference.removePreference( + findPreference(getString(R.string.pref_key_camerax_rear_camera_target_resolution))); + cameraPreference.removePreference( + findPreference(getString(R.string.pref_key_camerax_front_camera_target_resolution))); + setUpCameraPreviewSizePreference( + R.string.pref_key_rear_camera_preview_size, + R.string.pref_key_rear_camera_picture_size, + CameraSource.CAMERA_FACING_BACK); + setUpCameraPreviewSizePreference( + R.string.pref_key_front_camera_preview_size, + R.string.pref_key_front_camera_picture_size, + CameraSource.CAMERA_FACING_FRONT); + } + + private void setUpCameraPreviewSizePreference( + @StringRes int previewSizePrefKeyId, @StringRes int pictureSizePrefKeyId, int cameraId) { + ListPreference previewSizePreference = + (ListPreference) findPreference(getString(previewSizePrefKeyId)); + + Camera camera = null; + try { + camera = Camera.open(cameraId); + + List previewSizeList = CameraSource.generateValidPreviewSizeList(camera); + String[] previewSizeStringValues = new String[previewSizeList.size()]; + Map previewToPictureSizeStringMap = new HashMap<>(); + for (int i = 0; i < previewSizeList.size(); i++) { + CameraSource.SizePair sizePair = previewSizeList.get(i); + previewSizeStringValues[i] = sizePair.preview.toString(); + if (sizePair.picture != null) { + previewToPictureSizeStringMap.put( + sizePair.preview.toString(), sizePair.picture.toString()); + } + } + previewSizePreference.setEntries(previewSizeStringValues); + previewSizePreference.setEntryValues(previewSizeStringValues); + + if (previewSizePreference.getEntry() == null) { + CameraSource.SizePair sizePair = + CameraSource.selectSizePair( + camera, + CameraSource.DEFAULT_REQUESTED_CAMERA_PREVIEW_WIDTH, + CameraSource.DEFAULT_REQUESTED_CAMERA_PREVIEW_HEIGHT); + String previewSizeString = sizePair.preview.toString(); + previewSizePreference.setValue(previewSizeString); + previewSizePreference.setSummary(previewSizeString); + PreferenceUtils.saveString( + getActivity(), + pictureSizePrefKeyId, + sizePair.picture != null ? sizePair.picture.toString() : null); + } else { + previewSizePreference.setSummary(previewSizePreference.getEntry()); + } + + previewSizePreference.setOnPreferenceChangeListener( + (preference, newValue) -> { + String newPreviewSizeStringValue = (String) newValue; + previewSizePreference.setSummary(newPreviewSizeStringValue); + PreferenceUtils.saveString( + getActivity(), + pictureSizePrefKeyId, + previewToPictureSizeStringMap.get(newPreviewSizeStringValue)); + return true; + }); + } catch (RuntimeException e) { + ((PreferenceCategory) findPreference(getString(R.string.pref_category_key_camera))) + .removePreference(previewSizePreference); + } finally { + if (camera != null) { + camera.release(); + } + } + } + + +} diff --git a/app/src/main/java/com/modarb/android/posedetection/CameraSettingsActivity.java b/app/src/main/java/com/modarb/android/posedetection/CameraSettingsActivity.java new file mode 100644 index 0000000..5dcef3c --- /dev/null +++ b/app/src/main/java/com/modarb/android/posedetection/CameraSettingsActivity.java @@ -0,0 +1,55 @@ + +package com.modarb.android.posedetection; + + +import android.os.Bundle; +import android.preference.PreferenceFragment; + +import androidx.appcompat.app.ActionBar; +import androidx.appcompat.app.AppCompatActivity; + +import com.modarb.android.R; + + +public class CameraSettingsActivity extends AppCompatActivity { + + public static final String EXTRA_LAUNCH_SOURCE = "extra_launch_source"; + + @Override + protected void onCreate(Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + + setContentView(R.layout.activity_camera_setting); + + LaunchSource launchSource = + (LaunchSource) getIntent().getSerializableExtra(EXTRA_LAUNCH_SOURCE); + ActionBar actionBar = getSupportActionBar(); + if (actionBar != null) { + actionBar.setTitle(launchSource.titleResId); + } + + try { + assert launchSource != null; + getFragmentManager() + .beginTransaction() + .replace( + R.id.settings_container, + launchSource.prefFragmentClass.getDeclaredConstructor().newInstance()) + .commit(); + } catch (Exception e) { + throw new RuntimeException(e); + } + } + + public enum LaunchSource { + LIVE_PREVIEW(R.string.pref_screen_title_live_preview, CameraPreferenceFragment.class); + + private final int titleResId; + private final Class prefFragmentClass; + + LaunchSource(int titleResId, Class prefFragmentClass) { + this.titleResId = titleResId; + this.prefFragmentClass = prefFragmentClass; + } + } +} diff --git a/app/src/main/java/com/modarb/android/posedetection/FrameMetadata.java b/app/src/main/java/com/modarb/android/posedetection/FrameMetadata.java new file mode 100644 index 0000000..b02a031 --- /dev/null +++ b/app/src/main/java/com/modarb/android/posedetection/FrameMetadata.java @@ -0,0 +1,68 @@ +/* + * Copyright 2020 Google LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.modarb.android.posedetection; + +public class FrameMetadata { + + private final int width; + private final int height; + private final int rotation; + + private FrameMetadata(int width, int height, int rotation) { + this.width = width; + this.height = height; + this.rotation = rotation; + } + + public int getWidth() { + return width; + } + + public int getHeight() { + return height; + } + + public int getRotation() { + return rotation; + } + + public static class Builder { + + private int width; + private int height; + private int rotation; + + public Builder setWidth(int width) { + this.width = width; + return this; + } + + public Builder setHeight(int height) { + this.height = height; + return this; + } + + public Builder setRotation(int rotation) { + this.rotation = rotation; + return this; + } + + public FrameMetadata build() { + return new FrameMetadata(width, height, rotation); + } + } +} diff --git a/app/src/main/java/com/modarb/android/posedetection/GraphicOverlay.java b/app/src/main/java/com/modarb/android/posedetection/GraphicOverlay.java new file mode 100644 index 0000000..0ba6a95 --- /dev/null +++ b/app/src/main/java/com/modarb/android/posedetection/GraphicOverlay.java @@ -0,0 +1,244 @@ +/* + * Copyright 2020 Google LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.modarb.android.posedetection; + +import static java.lang.Math.max; +import static java.lang.Math.min; + +import android.content.Context; +import android.graphics.Canvas; +import android.graphics.Matrix; +import android.graphics.Paint; +import android.util.AttributeSet; +import android.view.View; + +import com.google.common.base.Preconditions; +import com.google.common.primitives.Ints; + +import java.util.ArrayList; +import java.util.List; + + +public class GraphicOverlay extends View { + private final Object lock = new Object(); + private final List graphics = new ArrayList<>(); + private final Matrix transformationMatrix = new Matrix(); + + private int imageWidth; + private int imageHeight; + + private float scaleFactor = 1.0f; + + private float postScaleWidthOffset; + + private float postScaleHeightOffset; + private boolean isImageFlipped; + private boolean needUpdateTransformation = true; + + + public GraphicOverlay(Context context, AttributeSet attrs) { + super(context, attrs); + addOnLayoutChangeListener( + (view, left, top, right, bottom, oldLeft, oldTop, oldRight, oldBottom) -> + needUpdateTransformation = true); + } + + /** + * Removes all graphics from the overlay. + */ + public void clear() { + synchronized (lock) { + graphics.clear(); + } + postInvalidate(); + } + + /** + * Adds a graphic to the overlay. + */ + public void add(Graphic graphic) { + synchronized (lock) { + graphics.add(graphic); + } + } + + /** + * Removes a graphic from the overlay. + */ + public void remove(Graphic graphic) { + synchronized (lock) { + graphics.remove(graphic); + } + postInvalidate(); + } + + public void setImageSourceInfo(int imageWidth, int imageHeight, boolean isFlipped) { + Preconditions.checkState(imageWidth > 0, "image width must be positive"); + Preconditions.checkState(imageHeight > 0, "image height must be positive"); + synchronized (lock) { + this.imageWidth = imageWidth; + this.imageHeight = imageHeight; + this.isImageFlipped = isFlipped; + needUpdateTransformation = true; + } + postInvalidate(); + } + + public int getImageWidth() { + return imageWidth; + } + + public int getImageHeight() { + return imageHeight; + } + + private void updateTransformationIfNeeded() { + if (!needUpdateTransformation || imageWidth <= 0 || imageHeight <= 0) { + return; + } + float viewAspectRatio = (float) getWidth() / getHeight(); + float imageAspectRatio = (float) imageWidth / imageHeight; + postScaleWidthOffset = 0; + postScaleHeightOffset = 0; + if (viewAspectRatio > imageAspectRatio) { + scaleFactor = (float) getWidth() / imageWidth; + postScaleHeightOffset = ((float) getWidth() / imageAspectRatio - getHeight()) / 2; + } else { + scaleFactor = (float) getHeight() / imageHeight; + postScaleWidthOffset = ((float) getHeight() * imageAspectRatio - getWidth()) / 2; + } + + transformationMatrix.reset(); + transformationMatrix.setScale(scaleFactor, scaleFactor); + transformationMatrix.postTranslate(-postScaleWidthOffset, -postScaleHeightOffset); + + if (isImageFlipped) { + transformationMatrix.postScale(-1f, 1f, getWidth() / 2f, getHeight() / 2f); + } + + needUpdateTransformation = false; + } + + @Override + protected void onDraw(Canvas canvas) { + super.onDraw(canvas); + + synchronized (lock) { + updateTransformationIfNeeded(); + + for (Graphic graphic : graphics) { + graphic.draw(canvas); + } + } + } + + public abstract static class Graphic { + private GraphicOverlay overlay; + + public Graphic(GraphicOverlay overlay) { + this.overlay = overlay; + } + + + public abstract void draw(Canvas canvas); + + protected void drawRect( + Canvas canvas, float left, float top, float right, float bottom, Paint paint) { + canvas.drawRect(left, top, right, bottom, paint); + } + + protected void drawText(Canvas canvas, String text, float x, float y, Paint paint) { + canvas.drawText(text, x, y, paint); + } + + public float scale(float imagePixel) { + return imagePixel * overlay.scaleFactor; + } + + + public Context getApplicationContext() { + return overlay.getContext().getApplicationContext(); + } + + public boolean isImageFlipped() { + return overlay.isImageFlipped; + } + + + public float translateX(float x) { + if (overlay.isImageFlipped) { + return overlay.getWidth() - (scale(x) - overlay.postScaleWidthOffset); + } else { + return scale(x) - overlay.postScaleWidthOffset; + } + } + + + public float translateY(float y) { + return scale(y) - overlay.postScaleHeightOffset; + } + + + public Matrix getTransformationMatrix() { + return overlay.transformationMatrix; + } + + public void postInvalidate() { + overlay.postInvalidate(); + } + + + public void updatePaintColorByZValue( + Paint paint, + Canvas canvas, + boolean visualizeZ, + boolean rescaleZForVisualization, + float zInImagePixel, + float zMin, + float zMax) { + if (!visualizeZ) { + return; + } + + float zLowerBoundInScreenPixel; + float zUpperBoundInScreenPixel; + + if (rescaleZForVisualization) { + zLowerBoundInScreenPixel = min(-0.001f, scale(zMin)); + zUpperBoundInScreenPixel = max(0.001f, scale(zMax)); + } else { + float defaultRangeFactor = 1f; + zLowerBoundInScreenPixel = -defaultRangeFactor * canvas.getWidth(); + zUpperBoundInScreenPixel = defaultRangeFactor * canvas.getWidth(); + } + + float zInScreenPixel = scale(zInImagePixel); + + if (zInScreenPixel < 0) { + + int v = (int) (zInScreenPixel / zLowerBoundInScreenPixel * 255); + v = Ints.constrainToRange(v, 0, 255); + paint.setARGB(255, 255, 255 - v, 255 - v); + } else { + + int v = (int) (zInScreenPixel / zUpperBoundInScreenPixel * 255); + v = Ints.constrainToRange(v, 0, 255); + paint.setARGB(255, 255 - v, 255 - v, 255); + } + } + } +} diff --git a/app/src/main/java/com/modarb/android/posedetection/RequestPermissionsActivity.kt b/app/src/main/java/com/modarb/android/posedetection/RequestPermissionsActivity.kt new file mode 100644 index 0000000..80fd161 --- /dev/null +++ b/app/src/main/java/com/modarb/android/posedetection/RequestPermissionsActivity.kt @@ -0,0 +1,97 @@ +package com.modarb.android.posedetection + +import android.Manifest +import android.content.Context +import android.content.Intent +import android.content.pm.PackageManager +import android.os.Bundle +import android.util.Log +import androidx.appcompat.app.AppCompatActivity +import androidx.core.app.ActivityCompat +import androidx.core.content.ContextCompat +import com.modarb.android.R +import com.modarb.android.posedetection.Utils.PermissionResultCallback + +class RequestPermissionsActivity : AppCompatActivity(), + ActivityCompat.OnRequestPermissionsResultCallback, PermissionResultCallback { + + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + setContentView(R.layout.activity_request_permissions) + + if (!allRuntimePermissionsGranted()) { + getRuntimePermissions() + } else { + onPermissionGranted() + } + } + + override fun onPermissionGranted() { + Log.i("Great", "All permissions granted") + startActivity(Intent(this, CameraActivity::class.java)) + } + + private fun allRuntimePermissionsGranted(): Boolean { + for (permission in REQUIRED_RUNTIME_PERMISSIONS) { + permission.let { + if (!isPermissionGranted(this, it)) { + return false + } + } + } + return true + } + + private fun getRuntimePermissions() { + val permissionsToRequest = ArrayList() + for (permission in REQUIRED_RUNTIME_PERMISSIONS) { + permission.let { + if (!isPermissionGranted(this, it)) { + permissionsToRequest.add(permission) + } + } + } + + if (permissionsToRequest.isNotEmpty()) { + ActivityCompat.requestPermissions( + this, permissionsToRequest.toTypedArray(), PERMISSION_REQUESTS + ) + } else { + onPermissionGranted() + } + } + + private fun isPermissionGranted(context: Context, permission: String): Boolean { + if (ContextCompat.checkSelfPermission( + context, permission + ) == PackageManager.PERMISSION_GRANTED + ) { + return true + } + Log.i("PERM", "Permission NOT granted: $permission") + return false + } + + override fun onRequestPermissionsResult( + requestCode: Int, permissions: Array, grantResults: IntArray + ) { + super.onRequestPermissionsResult(requestCode, permissions, grantResults) + if (requestCode == PERMISSION_REQUESTS) { + if (allRuntimePermissionsGranted()) { + onPermissionGranted() + } + } + } + + companion object { + private const val TAG = "RequestPermissionsActivity" + private const val PERMISSION_REQUESTS = 1 + + private val REQUIRED_RUNTIME_PERMISSIONS = arrayOf( + Manifest.permission.CAMERA, + Manifest.permission.WRITE_EXTERNAL_STORAGE, + Manifest.permission.READ_EXTERNAL_STORAGE + ) + } +} + diff --git a/app/src/main/java/com/modarb/android/posedetection/Utils/BitmapUtils.java b/app/src/main/java/com/modarb/android/posedetection/Utils/BitmapUtils.java new file mode 100644 index 0000000..3b13c0f --- /dev/null +++ b/app/src/main/java/com/modarb/android/posedetection/Utils/BitmapUtils.java @@ -0,0 +1,241 @@ +/* + * Copyright 2020 Google LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.modarb.android.posedetection.Utils; + +import android.content.ContentResolver; +import android.graphics.Bitmap; +import android.graphics.BitmapFactory; +import android.graphics.ImageFormat; +import android.graphics.Matrix; +import android.graphics.Rect; +import android.graphics.YuvImage; +import android.media.Image.Plane; +import android.net.Uri; +import android.os.Build.VERSION_CODES; +import android.provider.MediaStore; +import android.util.Log; + +import androidx.annotation.Nullable; +import androidx.annotation.RequiresApi; +import androidx.camera.core.ExperimentalGetImage; +import androidx.camera.core.ImageProxy; +import androidx.exifinterface.media.ExifInterface; + +import com.modarb.android.posedetection.FrameMetadata; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.nio.ByteBuffer; + +public class BitmapUtils { + private static final String TAG = "BitmapUtils"; + + @Nullable + public static Bitmap getBitmap(ByteBuffer data, FrameMetadata metadata) { + data.rewind(); + byte[] imageInBuffer = new byte[data.limit()]; + data.get(imageInBuffer, 0, imageInBuffer.length); + try { + YuvImage image = + new YuvImage( + imageInBuffer, ImageFormat.NV21, metadata.getWidth(), metadata.getHeight(), null); + ByteArrayOutputStream stream = new ByteArrayOutputStream(); + image.compressToJpeg(new Rect(0, 0, metadata.getWidth(), metadata.getHeight()), 80, stream); + + Bitmap bmp = BitmapFactory.decodeByteArray(stream.toByteArray(), 0, stream.size()); + + stream.close(); + return rotateBitmap(bmp, metadata.getRotation(), false, false); + } catch (Exception e) { + Log.e("VisionProcessorBase", "Error: " + e.getMessage()); + } + return null; + } + + @RequiresApi(VERSION_CODES.LOLLIPOP) + @Nullable + @ExperimentalGetImage + public static Bitmap getBitmap(ImageProxy image) { + FrameMetadata frameMetadata = + new FrameMetadata.Builder() + .setWidth(image.getWidth()) + .setHeight(image.getHeight()) + .setRotation(image.getImageInfo().getRotationDegrees()) + .build(); + + ByteBuffer nv21Buffer = + yuv420ThreePlanesToNV21(image.getImage().getPlanes(), image.getWidth(), image.getHeight()); + return getBitmap(nv21Buffer, frameMetadata); + } + + private static Bitmap rotateBitmap( + Bitmap bitmap, int rotationDegrees, boolean flipX, boolean flipY) { + Matrix matrix = new Matrix(); + + matrix.postRotate(rotationDegrees); + + matrix.postScale(flipX ? -1.0f : 1.0f, flipY ? -1.0f : 1.0f); + Bitmap rotatedBitmap = + Bitmap.createBitmap(bitmap, 0, 0, bitmap.getWidth(), bitmap.getHeight(), matrix, true); + + if (rotatedBitmap != bitmap) { + bitmap.recycle(); + } + return rotatedBitmap; + } + + @Nullable + public static Bitmap getBitmapFromContentUri(ContentResolver contentResolver, Uri imageUri) + throws IOException { + Bitmap decodedBitmap = MediaStore.Images.Media.getBitmap(contentResolver, imageUri); + if (decodedBitmap == null) { + return null; + } + int orientation = getExifOrientationTag(contentResolver, imageUri); + + int rotationDegrees = 0; + boolean flipX = false; + boolean flipY = false; + + switch (orientation) { + case ExifInterface.ORIENTATION_FLIP_HORIZONTAL: + flipX = true; + break; + case ExifInterface.ORIENTATION_ROTATE_90: + rotationDegrees = 90; + break; + case ExifInterface.ORIENTATION_TRANSPOSE: + rotationDegrees = 90; + flipX = true; + break; + case ExifInterface.ORIENTATION_ROTATE_180: + rotationDegrees = 180; + break; + case ExifInterface.ORIENTATION_FLIP_VERTICAL: + flipY = true; + break; + case ExifInterface.ORIENTATION_ROTATE_270: + rotationDegrees = -90; + break; + case ExifInterface.ORIENTATION_TRANSVERSE: + rotationDegrees = -90; + flipX = true; + break; + case ExifInterface.ORIENTATION_UNDEFINED: + case ExifInterface.ORIENTATION_NORMAL: + default: + } + + return rotateBitmap(decodedBitmap, rotationDegrees, flipX, flipY); + } + + private static int getExifOrientationTag(ContentResolver resolver, Uri imageUri) { + + if (!ContentResolver.SCHEME_CONTENT.equals(imageUri.getScheme()) + && !ContentResolver.SCHEME_FILE.equals(imageUri.getScheme())) { + return 0; + } + + ExifInterface exif; + try (InputStream inputStream = resolver.openInputStream(imageUri)) { + if (inputStream == null) { + return 0; + } + + exif = new ExifInterface(inputStream); + } catch (IOException e) { + Log.e(TAG, "failed to open file to read rotation meta data: " + imageUri, e); + return 0; + } + + return exif.getAttributeInt(ExifInterface.TAG_ORIENTATION, ExifInterface.ORIENTATION_NORMAL); + } + + + private static ByteBuffer yuv420ThreePlanesToNV21( + Plane[] yuv420888planes, int width, int height) { + int imageSize = width * height; + byte[] out = new byte[imageSize + 2 * (imageSize / 4)]; + + if (areUVPlanesNV21(yuv420888planes, width, height)) { + // Copy the Y values. + yuv420888planes[0].getBuffer().get(out, 0, imageSize); + + ByteBuffer uBuffer = yuv420888planes[1].getBuffer(); + ByteBuffer vBuffer = yuv420888planes[2].getBuffer(); + vBuffer.get(out, imageSize, 1); + uBuffer.get(out, imageSize + 1, 2 * imageSize / 4 - 1); + } else { + + unpackPlane(yuv420888planes[0], width, height, out, 0, 1); + unpackPlane(yuv420888planes[1], width, height, out, imageSize + 1, 2); + unpackPlane(yuv420888planes[2], width, height, out, imageSize, 2); + } + + return ByteBuffer.wrap(out); + } + + + private static boolean areUVPlanesNV21(Plane[] planes, int width, int height) { + int imageSize = width * height; + + ByteBuffer uBuffer = planes[1].getBuffer(); + ByteBuffer vBuffer = planes[2].getBuffer(); + + int vBufferPosition = vBuffer.position(); + int uBufferLimit = uBuffer.limit(); + + vBuffer.position(vBufferPosition + 1); + uBuffer.limit(uBufferLimit - 1); + + boolean areNV21 = + (vBuffer.remaining() == (2 * imageSize / 4 - 2)) && (vBuffer.compareTo(uBuffer) == 0); + + vBuffer.position(vBufferPosition); + uBuffer.limit(uBufferLimit); + + return areNV21; + } + + private static void unpackPlane( + Plane plane, int width, int height, byte[] out, int offset, int pixelStride) { + ByteBuffer buffer = plane.getBuffer(); + buffer.rewind(); + + + int numRow = (buffer.limit() + plane.getRowStride() - 1) / plane.getRowStride(); + if (numRow == 0) { + return; + } + int scaleFactor = height / numRow; + int numCol = width / scaleFactor; + + + int outputPos = offset; + int rowStart = 0; + for (int row = 0; row < numRow; row++) { + int inputPos = rowStart; + for (int col = 0; col < numCol; col++) { + out[outputPos] = buffer.get(inputPos); + outputPos += pixelStride; + inputPos += plane.getPixelStride(); + } + rowStart += plane.getRowStride(); + } + } +} diff --git a/app/src/main/java/com/modarb/android/posedetection/Utils/CameraImageGraphic.java b/app/src/main/java/com/modarb/android/posedetection/Utils/CameraImageGraphic.java new file mode 100644 index 0000000..fdc5148 --- /dev/null +++ b/app/src/main/java/com/modarb/android/posedetection/Utils/CameraImageGraphic.java @@ -0,0 +1,37 @@ +/* + * Copyright 2020 Google LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.modarb.android.posedetection.Utils; + +import android.graphics.Bitmap; +import android.graphics.Canvas; + +import com.modarb.android.posedetection.GraphicOverlay; + +public class CameraImageGraphic extends GraphicOverlay.Graphic { + + private final Bitmap bitmap; + + public CameraImageGraphic(GraphicOverlay overlay, Bitmap bitmap) { + super(overlay); + this.bitmap = bitmap; + } + + @Override + public void draw(Canvas canvas) { + canvas.drawBitmap(bitmap, getTransformationMatrix(), null); + } +} diff --git a/app/src/main/java/com/modarb/android/posedetection/Utils/CameraSource.java b/app/src/main/java/com/modarb/android/posedetection/Utils/CameraSource.java new file mode 100644 index 0000000..83a9887 --- /dev/null +++ b/app/src/main/java/com/modarb/android/posedetection/Utils/CameraSource.java @@ -0,0 +1,550 @@ +/* + * Copyright 2020 Google LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.modarb.android.posedetection.Utils; + +import android.Manifest; +import android.annotation.SuppressLint; +import android.app.Activity; +import android.content.Context; +import android.graphics.ImageFormat; +import android.graphics.SurfaceTexture; +import android.hardware.Camera; +import android.hardware.Camera.CameraInfo; +import android.hardware.Camera.Parameters; +import android.util.Log; +import android.view.Surface; +import android.view.SurfaceHolder; +import android.view.WindowManager; + +import androidx.annotation.Nullable; +import androidx.annotation.RequiresPermission; + +import com.google.android.gms.common.images.Size; +import com.modarb.android.posedetection.FrameMetadata; +import com.modarb.android.posedetection.GraphicOverlay; +import com.modarb.android.posedetection.VisionImageProcessor; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.IdentityHashMap; +import java.util.List; + +/** + * Manages the camera and allows UI updates on top of it (e.g. overlaying extra Graphics or + * displaying extra information). This receives preview frames from the camera at a specified rate, + * sending those frames to child classes' detectors / classifiers as fast as it is able to process. + */ +public class CameraSource { + @SuppressLint("InlinedApi") + public static final int CAMERA_FACING_BACK = CameraInfo.CAMERA_FACING_BACK; + + @SuppressLint("InlinedApi") + public static final int CAMERA_FACING_FRONT = CameraInfo.CAMERA_FACING_FRONT; + + public static final int IMAGE_FORMAT = ImageFormat.NV21; + public static final int DEFAULT_REQUESTED_CAMERA_PREVIEW_WIDTH = 480; + public static final int DEFAULT_REQUESTED_CAMERA_PREVIEW_HEIGHT = 360; + + private static final String TAG = "MIDemoApp:CameraSource"; + + /** + * The dummy surface texture must be assigned a chosen name. Since we never use an OpenGL context, + * we can choose any ID we want here. The dummy surface texture is not a crazy hack - it is + * actually how the camera team recommends using the camera without a preview. + */ + private static final int DUMMY_TEXTURE_NAME = 100; + + /** + * If the absolute difference between a preview size aspect ratio and a picture size aspect ratio + * is less than this tolerance, they are considered to be the same aspect ratio. + */ + private static final float ASPECT_RATIO_TOLERANCE = 0.01f; + private static final float REQUESTED_FPS = 30.0f; + private static final boolean REQUESTED_AUTO_FOCUS = true; + private final GraphicOverlay graphicOverlay; + private final FrameProcessingRunnable processingRunnable; + private final Object processorLock = new Object(); + private final IdentityHashMap bytesToByteBuffer = new IdentityHashMap<>(); + protected Activity activity; + private Camera camera; + private int facing = CAMERA_FACING_BACK; + /** + * Rotation of the device, and thus the associated preview images captured from the device. + */ + private int rotationDegrees; + private Size previewSize; + private SurfaceTexture dummySurfaceTexture; + /** + * Dedicated thread and associated runnable for calling into the detector with frames, as the + * frames become available from the camera. + */ + private Thread processingThread; + private VisionImageProcessor frameProcessor; + + public CameraSource(Activity activity, GraphicOverlay overlay) { + this.activity = activity; + graphicOverlay = overlay; + graphicOverlay.clear(); + processingRunnable = new FrameProcessingRunnable(); + } + + private static int getZoomValue(Parameters params, float zoomRatio) { + int zoom = (int) (Math.max(zoomRatio, 1) * 100); + List zoomRatios = params.getZoomRatios(); + int maxZoom = params.getMaxZoom(); + for (int i = 0; i < maxZoom; ++i) { + if (zoomRatios.get(i + 1) > zoom) { + return i; + } + } + return maxZoom; + } + + private static int getIdForRequestedCamera(int facing) { + CameraInfo cameraInfo = new CameraInfo(); + for (int i = 0; i < Camera.getNumberOfCameras(); ++i) { + Camera.getCameraInfo(i, cameraInfo); + if (cameraInfo.facing == facing) { + return i; + } + } + return -1; + } + + public static SizePair selectSizePair(Camera camera, int desiredWidth, int desiredHeight) { + List validPreviewSizes = generateValidPreviewSizeList(camera); + + SizePair selectedPair = null; + int minDiff = Integer.MAX_VALUE; + for (SizePair sizePair : validPreviewSizes) { + Size size = sizePair.preview; + int diff = + Math.abs(size.getWidth() - desiredWidth) + Math.abs(size.getHeight() - desiredHeight); + if (diff < minDiff) { + selectedPair = sizePair; + minDiff = diff; + } + } + + return selectedPair; + } + + public static List generateValidPreviewSizeList(Camera camera) { + Parameters parameters = camera.getParameters(); + List supportedPreviewSizes = parameters.getSupportedPreviewSizes(); + List supportedPictureSizes = parameters.getSupportedPictureSizes(); + List validPreviewSizes = new ArrayList<>(); + for (Camera.Size previewSize : supportedPreviewSizes) { + float previewAspectRatio = (float) previewSize.width / (float) previewSize.height; + + // By looping through the picture sizes in order, we favor the higher resolutions. + // We choose the highest resolution in order to support taking the full resolution + // picture later. + for (Camera.Size pictureSize : supportedPictureSizes) { + float pictureAspectRatio = (float) pictureSize.width / (float) pictureSize.height; + if (Math.abs(previewAspectRatio - pictureAspectRatio) < ASPECT_RATIO_TOLERANCE) { + validPreviewSizes.add(new SizePair(previewSize, pictureSize)); + break; + } + } + } + + if (validPreviewSizes.size() == 0) { + Log.w(TAG, "No preview sizes have a corresponding same-aspect-ratio picture size"); + for (Camera.Size previewSize : supportedPreviewSizes) { + // The null picture size will let us know that we shouldn't set a picture size. + validPreviewSizes.add(new SizePair(previewSize, null)); + } + } + + return validPreviewSizes; + } + + @SuppressLint("InlinedApi") + private static int[] selectPreviewFpsRange(Camera camera, float desiredPreviewFps) { + + int desiredPreviewFpsScaled = (int) (desiredPreviewFps * 1000.0f); + + + int[] selectedFpsRange = null; + int minUpperBoundDiff = Integer.MAX_VALUE; + int minLowerBound = Integer.MAX_VALUE; + List previewFpsRangeList = camera.getParameters().getSupportedPreviewFpsRange(); + for (int[] range : previewFpsRangeList) { + int upperBoundDiff = + Math.abs(desiredPreviewFpsScaled - range[Parameters.PREVIEW_FPS_MAX_INDEX]); + int lowerBound = range[Parameters.PREVIEW_FPS_MIN_INDEX]; + if (upperBoundDiff <= minUpperBoundDiff && lowerBound <= minLowerBound) { + selectedFpsRange = range; + minUpperBoundDiff = upperBoundDiff; + minLowerBound = lowerBound; + } + } + return selectedFpsRange; + } + + public void release() { + synchronized (processorLock) { + stop(); + cleanScreen(); + + if (frameProcessor != null) { + frameProcessor.stop(); + } + } + } + + @RequiresPermission(Manifest.permission.CAMERA) + public synchronized CameraSource start() throws IOException { + if (camera != null) { + return this; + } + + camera = createCamera(); + dummySurfaceTexture = new SurfaceTexture(DUMMY_TEXTURE_NAME); + camera.setPreviewTexture(dummySurfaceTexture); + camera.startPreview(); + + processingThread = new Thread(processingRunnable); + processingRunnable.setActive(true); + processingThread.start(); + return this; + } + + @RequiresPermission(Manifest.permission.CAMERA) + public synchronized CameraSource start(SurfaceHolder surfaceHolder) throws IOException { + if (camera != null) { + return this; + } + + camera = createCamera(); + camera.setPreviewDisplay(surfaceHolder); + camera.startPreview(); + + processingThread = new Thread(processingRunnable); + processingRunnable.setActive(true); + processingThread.start(); + return this; + } + + public synchronized void stop() { + processingRunnable.setActive(false); + if (processingThread != null) { + try { + + processingThread.join(); + } catch (InterruptedException e) { + Log.d(TAG, "Frame processing thread interrupted on release."); + } + processingThread = null; + } + + if (camera != null) { + camera.stopPreview(); + camera.setPreviewCallbackWithBuffer(null); + try { + camera.setPreviewTexture(null); + dummySurfaceTexture = null; + camera.setPreviewDisplay(null); + } catch (Exception e) { + Log.e(TAG, "Failed to clear camera preview: " + e); + } + camera.release(); + camera = null; + } + + bytesToByteBuffer.clear(); + } + + public synchronized void setFacing(int facing) { + if ((facing != CAMERA_FACING_BACK) && (facing != CAMERA_FACING_FRONT)) { + throw new IllegalArgumentException("Invalid camera: " + facing); + } + this.facing = facing; + } + + public Size getPreviewSize() { + return previewSize; + } + + public int getCameraFacing() { + return facing; + } + + public boolean setZoom(float zoomRatio) { + Log.d(TAG, "setZoom: " + zoomRatio); + if (camera == null) { + return false; + } + + Parameters parameters = camera.getParameters(); + parameters.setZoom(getZoomValue(parameters, zoomRatio)); + camera.setParameters(parameters); + return true; + } + + @SuppressLint("InlinedApi") + private Camera createCamera() throws IOException { + int requestedCameraId = getIdForRequestedCamera(facing); + if (requestedCameraId == -1) { + throw new IOException("Could not find requested camera."); + } + Camera camera = Camera.open(requestedCameraId); + + SizePair sizePair = PreferenceUtils.getCameraPreviewSizePair(activity, requestedCameraId); + if (sizePair == null) { + sizePair = + selectSizePair( + camera, + DEFAULT_REQUESTED_CAMERA_PREVIEW_WIDTH, + DEFAULT_REQUESTED_CAMERA_PREVIEW_HEIGHT); + } + + if (sizePair == null) { + throw new IOException("Could not find suitable preview size."); + } + + previewSize = sizePair.preview; + Log.v(TAG, "Camera preview size: " + previewSize); + + int[] previewFpsRange = selectPreviewFpsRange(camera, REQUESTED_FPS); + if (previewFpsRange == null) { + throw new IOException("Could not find suitable preview frames per second range."); + } + + Parameters parameters = camera.getParameters(); + + Size pictureSize = sizePair.picture; + if (pictureSize != null) { + Log.v(TAG, "Camera picture size: " + pictureSize); + parameters.setPictureSize(pictureSize.getWidth(), pictureSize.getHeight()); + } + parameters.setPreviewSize(previewSize.getWidth(), previewSize.getHeight()); + parameters.setPreviewFpsRange( + previewFpsRange[Parameters.PREVIEW_FPS_MIN_INDEX], + previewFpsRange[Parameters.PREVIEW_FPS_MAX_INDEX]); + parameters.setPreviewFormat(IMAGE_FORMAT); + + setRotation(camera, parameters, requestedCameraId); + + if (REQUESTED_AUTO_FOCUS) { + if (parameters + .getSupportedFocusModes() + .contains(Parameters.FOCUS_MODE_CONTINUOUS_VIDEO)) { + parameters.setFocusMode(Parameters.FOCUS_MODE_CONTINUOUS_VIDEO); + } else { + Log.i(TAG, "Camera auto focus is not supported on this device."); + } + } + + camera.setParameters(parameters); + + + camera.setPreviewCallbackWithBuffer(new CameraPreviewCallback()); + camera.addCallbackBuffer(createPreviewBuffer(previewSize)); + camera.addCallbackBuffer(createPreviewBuffer(previewSize)); + camera.addCallbackBuffer(createPreviewBuffer(previewSize)); + camera.addCallbackBuffer(createPreviewBuffer(previewSize)); + + return camera; + } + + private void setRotation(Camera camera, Parameters parameters, int cameraId) { + WindowManager windowManager = (WindowManager) activity.getSystemService(Context.WINDOW_SERVICE); + int degrees = 0; + int rotation = windowManager.getDefaultDisplay().getRotation(); + switch (rotation) { + case Surface.ROTATION_0: + degrees = 0; + break; + case Surface.ROTATION_90: + degrees = 90; + break; + case Surface.ROTATION_180: + degrees = 180; + break; + case Surface.ROTATION_270: + degrees = 270; + break; + default: + Log.e(TAG, "Bad rotation value: " + rotation); + } + + CameraInfo cameraInfo = new CameraInfo(); + Camera.getCameraInfo(cameraId, cameraInfo); + + int displayAngle; + if (cameraInfo.facing == CameraInfo.CAMERA_FACING_FRONT) { + this.rotationDegrees = (cameraInfo.orientation + degrees) % 360; + displayAngle = (360 - this.rotationDegrees) % 360; + } else { + this.rotationDegrees = (cameraInfo.orientation - degrees + 360) % 360; + displayAngle = this.rotationDegrees; + } + Log.d(TAG, "Display rotation is: " + rotation); + Log.d(TAG, "Camera face is: " + cameraInfo.facing); + Log.d(TAG, "Camera rotation is: " + cameraInfo.orientation); + Log.d(TAG, "RotationDegrees is: " + this.rotationDegrees); + + camera.setDisplayOrientation(displayAngle); + parameters.setRotation(this.rotationDegrees); + } + + @SuppressLint("InlinedApi") + private byte[] createPreviewBuffer(Size previewSize) { + int bitsPerPixel = ImageFormat.getBitsPerPixel(IMAGE_FORMAT); + long sizeInBits = (long) previewSize.getHeight() * previewSize.getWidth() * bitsPerPixel; + int bufferSize = (int) Math.ceil(sizeInBits / 8.0d) + 1; + + + byte[] byteArray = new byte[bufferSize]; + ByteBuffer buffer = ByteBuffer.wrap(byteArray); + if (!buffer.hasArray() || (buffer.array() != byteArray)) { + + throw new IllegalStateException("Failed to create valid buffer for camera source."); + } + + bytesToByteBuffer.put(byteArray, buffer); + return byteArray; + } + + public void setMachineLearningFrameProcessor(VisionImageProcessor processor) { + synchronized (processorLock) { + cleanScreen(); + if (frameProcessor != null) { + frameProcessor.stop(); + } + frameProcessor = processor; + } + } + + private void cleanScreen() { + graphicOverlay.clear(); + } + + public static class SizePair { + public final Size preview; + @Nullable + public final Size picture; + + SizePair(Camera.Size previewSize, @Nullable Camera.Size pictureSize) { + preview = new Size(previewSize.width, previewSize.height); + picture = pictureSize != null ? new Size(pictureSize.width, pictureSize.height) : null; + } + + public SizePair(Size previewSize, @Nullable Size pictureSize) { + preview = previewSize; + picture = pictureSize; + } + } + + private class CameraPreviewCallback implements Camera.PreviewCallback { + @Override + public void onPreviewFrame(byte[] data, Camera camera) { + processingRunnable.setNextFrame(data, camera); + } + } + + private class FrameProcessingRunnable implements Runnable { + + private final Object lock = new Object(); + private boolean active = true; + + private ByteBuffer pendingFrameData; + + FrameProcessingRunnable() { + } + + void setActive(boolean active) { + synchronized (lock) { + this.active = active; + lock.notifyAll(); + } + } + + + @SuppressWarnings("ByteBufferBackingArray") + void setNextFrame(byte[] data, Camera camera) { + synchronized (lock) { + if (pendingFrameData != null) { + camera.addCallbackBuffer(pendingFrameData.array()); + pendingFrameData = null; + } + + if (!bytesToByteBuffer.containsKey(data)) { + Log.d( + TAG, + "Skipping frame. Could not find ByteBuffer associated with the image " + + "data from the camera."); + return; + } + + pendingFrameData = bytesToByteBuffer.get(data); + + lock.notifyAll(); + } + } + + @SuppressLint("InlinedApi") + @SuppressWarnings({"GuardedBy", "ByteBufferBackingArray"}) + @Override + public void run() { + ByteBuffer data; + + while (true) { + synchronized (lock) { + while (active && (pendingFrameData == null)) { + try { + + lock.wait(); + } catch (InterruptedException e) { + Log.d(TAG, "Frame processing loop terminated.", e); + return; + } + } + + if (!active) { + + return; + } + + data = pendingFrameData; + pendingFrameData = null; + } + + + try { + synchronized (processorLock) { + frameProcessor.processByteBuffer( + data, + new FrameMetadata.Builder() + .setWidth(previewSize.getWidth()) + .setHeight(previewSize.getHeight()) + .setRotation(rotationDegrees) + .build(), + graphicOverlay); + } + } catch (Exception t) { + Log.e(TAG, "Exception thrown from receiver.", t); + } finally { + camera.addCallbackBuffer(data.array()); + } + } + } + } +} diff --git a/app/src/main/java/com/modarb/android/posedetection/Utils/CameraSourcePreview.java b/app/src/main/java/com/modarb/android/posedetection/Utils/CameraSourcePreview.java new file mode 100644 index 0000000..173a24b --- /dev/null +++ b/app/src/main/java/com/modarb/android/posedetection/Utils/CameraSourcePreview.java @@ -0,0 +1,181 @@ +/* + * Copyright 2020 Google LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.modarb.android.posedetection.Utils; + +import android.content.Context; +import android.content.res.Configuration; +import android.util.AttributeSet; +import android.util.Log; +import android.view.SurfaceHolder; +import android.view.SurfaceView; +import android.view.ViewGroup; + +import com.google.android.gms.common.images.Size; +import com.modarb.android.posedetection.GraphicOverlay; + +import java.io.IOException; + +/** + * Preview the camera image in the screen. + */ +public class CameraSourcePreview extends ViewGroup { + private static final String TAG = "MIDemoApp:Preview"; + + private final Context context; + private final SurfaceView surfaceView; + private boolean startRequested; + private boolean surfaceAvailable; + private CameraSource cameraSource; + + private GraphicOverlay overlay; + + public CameraSourcePreview(Context context, AttributeSet attrs) { + super(context, attrs); + this.context = context; + startRequested = false; + surfaceAvailable = false; + + surfaceView = new SurfaceView(context); + surfaceView.getHolder().addCallback(new SurfaceCallback()); + addView(surfaceView); + } + + private void start(CameraSource cameraSource) throws IOException { + this.cameraSource = cameraSource; + + if (this.cameraSource != null) { + startRequested = true; + startIfReady(); + } + } + + public void start(CameraSource cameraSource, GraphicOverlay overlay) throws IOException { + this.overlay = overlay; + start(cameraSource); + } + + public void stop() { + if (cameraSource != null) { + cameraSource.stop(); + } + } + + public void release() { + if (cameraSource != null) { + cameraSource.release(); + cameraSource = null; + } + surfaceView.getHolder().getSurface().release(); + } + + private void startIfReady() throws IOException, SecurityException { + if (startRequested && surfaceAvailable) { + if (PreferenceUtils.isCameraLiveViewportEnabled(context)) { + cameraSource.start(surfaceView.getHolder()); + } else { + cameraSource.start(); + } + requestLayout(); + + if (overlay != null) { + Size size = cameraSource.getPreviewSize(); + int min = Math.min(size.getWidth(), size.getHeight()); + int max = Math.max(size.getWidth(), size.getHeight()); + boolean isImageFlipped = cameraSource.getCameraFacing() == CameraSource.CAMERA_FACING_FRONT; + if (isPortraitMode()) { + // Swap width and height sizes when in portrait, since it will be rotated by 90 degrees. + // The camera preview and the image being processed have the same size. + overlay.setImageSourceInfo(min, max, isImageFlipped); + } else { + overlay.setImageSourceInfo(max, min, isImageFlipped); + } + overlay.clear(); + } + startRequested = false; + } + } + + @Override + protected void onLayout(boolean changed, int left, int top, int right, int bottom) { + int width = 320; + int height = 240; + if (cameraSource != null) { + Size size = cameraSource.getPreviewSize(); + if (size != null) { + width = size.getWidth(); + height = size.getHeight(); + } + } + + // Swap width and height sizes when in portrait, since it will be rotated 90 degrees + if (isPortraitMode()) { + int tmp = width; + width = height; + height = tmp; + } + + float previewAspectRatio = (float) width / height; + int layoutWidth = right - left; + int layoutHeight = bottom - top; + float layoutAspectRatio = (float) layoutWidth / layoutHeight; + if (previewAspectRatio > layoutAspectRatio) { + // The preview input is wider than the layout area. Fit the layout height and crop + // the preview input horizontally while keep the center. + int horizontalOffset = (int) (previewAspectRatio * layoutHeight - layoutWidth) / 2; + surfaceView.layout(-horizontalOffset, 0, layoutWidth + horizontalOffset, layoutHeight); + } else { + // The preview input is taller than the layout area. Fit the layout width and crop the preview + // input vertically while keep the center. + int verticalOffset = (int) (layoutWidth / previewAspectRatio - layoutHeight) / 2; + surfaceView.layout(0, -verticalOffset, layoutWidth, layoutHeight + verticalOffset); + } + } + + private boolean isPortraitMode() { + int orientation = context.getResources().getConfiguration().orientation; + if (orientation == Configuration.ORIENTATION_LANDSCAPE) { + return false; + } + if (orientation == Configuration.ORIENTATION_PORTRAIT) { + return true; + } + + Log.d(TAG, "isPortraitMode returning false by default"); + return false; + } + + private class SurfaceCallback implements SurfaceHolder.Callback { + @Override + public void surfaceCreated(SurfaceHolder surface) { + surfaceAvailable = true; + try { + startIfReady(); + } catch (IOException e) { + Log.e(TAG, "Could not start camera source.", e); + } + } + + @Override + public void surfaceDestroyed(SurfaceHolder surface) { + surfaceAvailable = false; + } + + @Override + public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) { + } + } +} diff --git a/app/src/main/java/com/modarb/android/posedetection/Utils/PermissionResultCallback.kt b/app/src/main/java/com/modarb/android/posedetection/Utils/PermissionResultCallback.kt new file mode 100644 index 0000000..9f6282a --- /dev/null +++ b/app/src/main/java/com/modarb/android/posedetection/Utils/PermissionResultCallback.kt @@ -0,0 +1,5 @@ +package com.modarb.android.posedetection.Utils + +interface PermissionResultCallback { + fun onPermissionGranted() +} diff --git a/app/src/main/java/com/modarb/android/posedetection/Utils/PreferenceUtils.java b/app/src/main/java/com/modarb/android/posedetection/Utils/PreferenceUtils.java new file mode 100644 index 0000000..93e61ee --- /dev/null +++ b/app/src/main/java/com/modarb/android/posedetection/Utils/PreferenceUtils.java @@ -0,0 +1,134 @@ +/* + * Copyright 2020 Google LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.modarb.android.posedetection.Utils; + +import android.content.Context; +import android.content.SharedPreferences; +import android.preference.PreferenceManager; + +import androidx.annotation.Nullable; +import androidx.annotation.StringRes; + +import com.google.android.gms.common.images.Size; +import com.google.common.base.Preconditions; +import com.google.mlkit.vision.pose.PoseDetectorOptionsBase; +import com.google.mlkit.vision.pose.accurate.AccuratePoseDetectorOptions; +import com.google.mlkit.vision.pose.defaults.PoseDetectorOptions; +import com.modarb.android.R; + +public class PreferenceUtils { + + private static final int POSE_DETECTOR_PERFORMANCE_MODE_FAST = 1; + + private PreferenceUtils() { + } + + public static void saveString(Context context, @StringRes int prefKeyId, @Nullable String value) { + PreferenceManager.getDefaultSharedPreferences(context) + .edit() + .putString(context.getString(prefKeyId), value) + .apply(); + } + + @Nullable + public static CameraSource.SizePair getCameraPreviewSizePair(Context context, int cameraId) { + Preconditions.checkArgument( + cameraId == CameraSource.CAMERA_FACING_BACK + || cameraId == CameraSource.CAMERA_FACING_FRONT); + String previewSizePrefKey; + String pictureSizePrefKey; + if (cameraId == CameraSource.CAMERA_FACING_BACK) { + previewSizePrefKey = context.getString(R.string.pref_key_rear_camera_preview_size); + pictureSizePrefKey = context.getString(R.string.pref_key_rear_camera_picture_size); + } else { + previewSizePrefKey = context.getString(R.string.pref_key_front_camera_preview_size); + pictureSizePrefKey = context.getString(R.string.pref_key_front_camera_picture_size); + } + + try { + SharedPreferences sharedPreferences = PreferenceManager.getDefaultSharedPreferences(context); + return new CameraSource.SizePair( + Size.parseSize(sharedPreferences.getString(previewSizePrefKey, null)), + Size.parseSize(sharedPreferences.getString(pictureSizePrefKey, null))); + } catch (Exception e) { + return null; + } + } + + public static PoseDetectorOptionsBase getPoseDetectorOptionsForLivePreview(Context context) { + int performanceMode = + getModeTypePreferenceValue( + context, + R.string.pref_key_live_preview_pose_detection_performance_mode, + POSE_DETECTOR_PERFORMANCE_MODE_FAST); + boolean preferGPU = preferGPUForPoseDetection(context); + if (performanceMode == POSE_DETECTOR_PERFORMANCE_MODE_FAST) { + PoseDetectorOptions.Builder builder = + new PoseDetectorOptions.Builder().setDetectorMode(PoseDetectorOptions.STREAM_MODE); + if (preferGPU) { + builder.setPreferredHardwareConfigs(PoseDetectorOptions.CPU_GPU); + } + return builder.build(); + } else { + AccuratePoseDetectorOptions.Builder builder = + new AccuratePoseDetectorOptions.Builder() + .setDetectorMode(AccuratePoseDetectorOptions.STREAM_MODE); + if (preferGPU) { + builder.setPreferredHardwareConfigs(AccuratePoseDetectorOptions.CPU_GPU); + } + return builder.build(); + } + } + + public static boolean preferGPUForPoseDetection(Context context) { + SharedPreferences sharedPreferences = PreferenceManager.getDefaultSharedPreferences(context); + String prefKey = context.getString(R.string.pref_key_pose_detector_prefer_gpu); + return sharedPreferences.getBoolean(prefKey, true); + } + + public static boolean shouldShowPoseDetectionInFrameLikelihoodLivePreview(Context context) { + SharedPreferences sharedPreferences = PreferenceManager.getDefaultSharedPreferences(context); + String prefKey = + context.getString(R.string.pref_key_live_preview_pose_detector_show_in_frame_likelihood); + return sharedPreferences.getBoolean(prefKey, true); + } + + public static boolean shouldPoseDetectionVisualizeZ(Context context) { + SharedPreferences sharedPreferences = PreferenceManager.getDefaultSharedPreferences(context); + String prefKey = context.getString(R.string.pref_key_pose_detector_visualize_z); + return sharedPreferences.getBoolean(prefKey, true); + } + + public static boolean shouldPoseDetectionRescaleZForVisualization(Context context) { + SharedPreferences sharedPreferences = PreferenceManager.getDefaultSharedPreferences(context); + String prefKey = context.getString(R.string.pref_key_pose_detector_rescale_z); + return sharedPreferences.getBoolean(prefKey, true); + } + + private static int getModeTypePreferenceValue( + Context context, @StringRes int prefKeyResId, int defaultValue) { + SharedPreferences sharedPreferences = PreferenceManager.getDefaultSharedPreferences(context); + String prefKey = context.getString(prefKeyResId); + return Integer.parseInt(sharedPreferences.getString(prefKey, String.valueOf(defaultValue))); + } + + public static boolean isCameraLiveViewportEnabled(Context context) { + SharedPreferences sharedPreferences = PreferenceManager.getDefaultSharedPreferences(context); + String prefKey = context.getString(R.string.pref_key_camera_live_viewport); + return sharedPreferences.getBoolean(prefKey, false); + } +} diff --git a/app/src/main/java/com/modarb/android/posedetection/Utils/ScopedExecutor.java b/app/src/main/java/com/modarb/android/posedetection/Utils/ScopedExecutor.java new file mode 100644 index 0000000..561c2a1 --- /dev/null +++ b/app/src/main/java/com/modarb/android/posedetection/Utils/ScopedExecutor.java @@ -0,0 +1,53 @@ +/* + * Copyright 2020 Google LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.modarb.android.posedetection.Utils; + +import androidx.annotation.NonNull; + +import java.util.concurrent.Executor; +import java.util.concurrent.atomic.AtomicBoolean; + + +public class ScopedExecutor implements Executor { + + private final Executor executor; + private final AtomicBoolean shutdown = new AtomicBoolean(); + + public ScopedExecutor(@NonNull Executor executor) { + this.executor = executor; + } + + @Override + public void execute(@NonNull Runnable command) { + if (shutdown.get()) { + return; + } + executor.execute( + () -> { + + if (shutdown.get()) { + return; + } + command.run(); + }); + } + + + public void shutdown() { + shutdown.set(true); + } +} diff --git a/app/src/main/java/com/modarb/android/posedetection/VisionImageProcessor.java b/app/src/main/java/com/modarb/android/posedetection/VisionImageProcessor.java new file mode 100644 index 0000000..45e0259 --- /dev/null +++ b/app/src/main/java/com/modarb/android/posedetection/VisionImageProcessor.java @@ -0,0 +1,38 @@ +/* + * Copyright 2020 Google LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.modarb.android.posedetection; + +import android.graphics.Bitmap; + +import androidx.camera.core.ImageProxy; + +import com.google.mlkit.common.MlKitException; + +import java.nio.ByteBuffer; + +public interface VisionImageProcessor { + + void processBitmap(Bitmap bitmap, GraphicOverlay graphicOverlay); + + void processByteBuffer( + ByteBuffer data, FrameMetadata frameMetadata, GraphicOverlay graphicOverlay) + throws MlKitException; + + void processImageProxy(ImageProxy image, GraphicOverlay graphicOverlay) throws MlKitException; + + void stop(); +} diff --git a/app/src/main/java/com/modarb/android/posedetection/VisionProcessorBase.kt b/app/src/main/java/com/modarb/android/posedetection/VisionProcessorBase.kt new file mode 100644 index 0000000..bcb1bb6 --- /dev/null +++ b/app/src/main/java/com/modarb/android/posedetection/VisionProcessorBase.kt @@ -0,0 +1,348 @@ +/* + * Copyright 2020 Google LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.google.mlkit.vision.demo.kotlin + +import android.app.ActivityManager +import android.content.Context +import android.graphics.Bitmap +import android.os.Build.VERSION_CODES +import android.os.SystemClock +import android.util.Log +import android.widget.Toast +import androidx.annotation.GuardedBy +import androidx.annotation.RequiresApi +import androidx.camera.core.ExperimentalGetImage +import androidx.camera.core.ImageProxy +import com.google.android.gms.tasks.OnFailureListener +import com.google.android.gms.tasks.OnSuccessListener +import com.google.android.gms.tasks.Task +import com.google.android.gms.tasks.TaskExecutors +import com.google.android.gms.tasks.Tasks +import com.google.android.odml.image.BitmapMlImageBuilder +import com.google.android.odml.image.ByteBufferMlImageBuilder +import com.google.android.odml.image.MediaMlImageBuilder +import com.google.android.odml.image.MlImage +import com.google.mlkit.common.MlKitException +import com.google.mlkit.vision.common.InputImage + +import com.modarb.android.posedetection.FrameMetadata +import com.modarb.android.posedetection.GraphicOverlay +import com.modarb.android.posedetection.Utils.BitmapUtils +import com.modarb.android.posedetection.Utils.CameraImageGraphic +import com.modarb.android.posedetection.Utils.PreferenceUtils +import com.modarb.android.posedetection.Utils.ScopedExecutor +import com.modarb.android.posedetection.VisionImageProcessor +import java.lang.Math.max +import java.lang.Math.min +import java.nio.ByteBuffer +import java.util.Timer +import java.util.TimerTask + + +abstract class VisionProcessorBase(context: Context) : VisionImageProcessor { + + companion object { + const val MANUAL_TESTING_LOG = "LogTagForTest" + private const val TAG = "VisionProcessorBase" + } + + private var activityManager: ActivityManager = + context.getSystemService(Context.ACTIVITY_SERVICE) as ActivityManager + private val fpsTimer = Timer() + private val executor = ScopedExecutor(TaskExecutors.MAIN_THREAD) + + private var isShutdown = false + + private var numRuns = 0 + private var totalFrameMs = 0L + private var maxFrameMs = 0L + private var minFrameMs = Long.MAX_VALUE + private var totalDetectorMs = 0L + private var maxDetectorMs = 0L + private var minDetectorMs = Long.MAX_VALUE + + private var frameProcessedInOneSecondInterval = 0 + private var framesPerSecond = 0 + + @GuardedBy("this") + private var latestImage: ByteBuffer? = null + + @GuardedBy("this") + private var latestImageMetaData: FrameMetadata? = null + + @GuardedBy("this") + private var processingImage: ByteBuffer? = null + + @GuardedBy("this") + private var processingMetaData: FrameMetadata? = null + + init { + fpsTimer.scheduleAtFixedRate( + object : TimerTask() { + override fun run() { + framesPerSecond = frameProcessedInOneSecondInterval + frameProcessedInOneSecondInterval = 0 + } + }, 0, 1000 + ) + } + + override fun processBitmap(bitmap: Bitmap?, graphicOverlay: GraphicOverlay) { + val frameStartMs = SystemClock.elapsedRealtime() + + if (isMlImageEnabled(graphicOverlay.context)) { + val mlImage = BitmapMlImageBuilder(bitmap!!).build() + requestDetectInImage( + mlImage, graphicOverlay, null, false, frameStartMs + ) + mlImage.close() + return + } + + requestDetectInImage( + InputImage.fromBitmap(bitmap!!, 0), graphicOverlay, null, false, frameStartMs + ) + } + + @Synchronized + override fun processByteBuffer( + data: ByteBuffer?, frameMetadata: FrameMetadata?, graphicOverlay: GraphicOverlay + ) { + latestImage = data + latestImageMetaData = frameMetadata + if (processingImage == null && processingMetaData == null) { + processLatestImage(graphicOverlay) + } + } + + @Synchronized + private fun processLatestImage(graphicOverlay: GraphicOverlay) { + processingImage = latestImage + processingMetaData = latestImageMetaData + latestImage = null + latestImageMetaData = null + if (processingImage != null && processingMetaData != null && !isShutdown) { + processImage(processingImage!!, processingMetaData!!, graphicOverlay) + } + } + + private fun processImage( + data: ByteBuffer, frameMetadata: FrameMetadata, graphicOverlay: GraphicOverlay + ) { + val frameStartMs = SystemClock.elapsedRealtime() + + val bitmap = if (PreferenceUtils.isCameraLiveViewportEnabled(graphicOverlay.context)) null + else BitmapUtils.getBitmap(data, frameMetadata) + + if (isMlImageEnabled(graphicOverlay.context)) { + val mlImage = ByteBufferMlImageBuilder( + data, frameMetadata.width, frameMetadata.height, MlImage.IMAGE_FORMAT_NV21 + ).setRotation(frameMetadata.rotation).build() + requestDetectInImage( + mlImage, graphicOverlay, bitmap, true, frameStartMs + ).addOnSuccessListener(executor) { processLatestImage(graphicOverlay) } + + mlImage.close() + return + } + + requestDetectInImage( + InputImage.fromByteBuffer( + data, + frameMetadata.width, + frameMetadata.height, + frameMetadata.rotation, + InputImage.IMAGE_FORMAT_NV21 + ), graphicOverlay, bitmap,/* shouldShowFps= */ true, frameStartMs + ).addOnSuccessListener(executor) { processLatestImage(graphicOverlay) } + } + + @RequiresApi(VERSION_CODES.LOLLIPOP) + @ExperimentalGetImage + override fun processImageProxy(image: ImageProxy, graphicOverlay: GraphicOverlay) { + val frameStartMs = SystemClock.elapsedRealtime() + if (isShutdown) { + return + } + var bitmap: Bitmap? = null + if (!PreferenceUtils.isCameraLiveViewportEnabled(graphicOverlay.context)) { + bitmap = BitmapUtils.getBitmap(image) + } + + if (isMlImageEnabled(graphicOverlay.context)) { + val mlImage = + MediaMlImageBuilder(image.image!!).setRotation(image.imageInfo.rotationDegrees) + .build() + requestDetectInImage( + mlImage, graphicOverlay,/* originalCameraImage= */ + bitmap,/* shouldShowFps= */ + true, frameStartMs + ) + // When the image is from CameraX analysis use case, must call image.close() on received + // images when finished using them. Otherwise, new images may not be received or the camera + // may stall. + // Currently MlImage doesn't support ImageProxy directly, so we still need to call + // ImageProxy.close() here. + .addOnCompleteListener { image.close() } + + return + } + + requestDetectInImage( + InputImage.fromMediaImage(image.image!!, image.imageInfo.rotationDegrees), + graphicOverlay, + bitmap, + true, + frameStartMs + ) + + .addOnCompleteListener { image.close() } + } + + private fun requestDetectInImage( + image: InputImage, + graphicOverlay: GraphicOverlay, + originalCameraImage: Bitmap?, + shouldShowFps: Boolean, + frameStartMs: Long + ): Task { + return setUpListener( + detectInImage(image), graphicOverlay, originalCameraImage, shouldShowFps, frameStartMs + ) + } + + private fun requestDetectInImage( + image: MlImage, + graphicOverlay: GraphicOverlay, + originalCameraImage: Bitmap?, + shouldShowFps: Boolean, + frameStartMs: Long + ): Task { + return setUpListener( + detectInImage(image), graphicOverlay, originalCameraImage, shouldShowFps, frameStartMs + ) + } + + private fun setUpListener( + task: Task, + graphicOverlay: GraphicOverlay, + originalCameraImage: Bitmap?, + shouldShowFps: Boolean, + frameStartMs: Long + ): Task { + val detectorStartMs = SystemClock.elapsedRealtime() + return task.addOnSuccessListener(executor, OnSuccessListener { results: T -> + val endMs = SystemClock.elapsedRealtime() + val currentFrameLatencyMs = endMs - frameStartMs + val currentDetectorLatencyMs = endMs - detectorStartMs + if (numRuns >= 500) { + resetLatencyStats() + } + numRuns++ + frameProcessedInOneSecondInterval++ + totalFrameMs += currentFrameLatencyMs + maxFrameMs = max(currentFrameLatencyMs, maxFrameMs) + minFrameMs = min(currentFrameLatencyMs, minFrameMs) + totalDetectorMs += currentDetectorLatencyMs + maxDetectorMs = max(currentDetectorLatencyMs, maxDetectorMs) + minDetectorMs = min(currentDetectorLatencyMs, minDetectorMs) + + + if (frameProcessedInOneSecondInterval == 1) { + Log.d(TAG, "Num of Runs: $numRuns") + Log.d( + TAG, + "Frame latency: max=" + maxFrameMs + ", min=" + minFrameMs + ", avg=" + totalFrameMs / numRuns + ) + Log.d( + TAG, + "Detector latency: max=" + maxDetectorMs + ", min=" + minDetectorMs + ", avg=" + totalDetectorMs / numRuns + ) + val mi = ActivityManager.MemoryInfo() + activityManager.getMemoryInfo(mi) + val availableMegs: Long = mi.availMem / 0x100000L + Log.d(TAG, "Memory available in system: $availableMegs MB") + } + graphicOverlay.clear() + if (originalCameraImage != null) { + graphicOverlay.add(CameraImageGraphic(graphicOverlay, originalCameraImage)) + } + this@VisionProcessorBase.onSuccess(results, graphicOverlay) + // TODO fix that +// if (!PreferenceUtils.shouldHideDetectionInfo(graphicOverlay.context)) { +// graphicOverlay.add( +// InferenceInfoGraphic( +// graphicOverlay, +// currentFrameLatencyMs, +// currentDetectorLatencyMs, +// if (shouldShowFps) framesPerSecond else null +// ) +// ) +// } + graphicOverlay.postInvalidate() + }).addOnFailureListener(executor, OnFailureListener { e: Exception -> + graphicOverlay.clear() + graphicOverlay.postInvalidate() + val error = "Failed to process. Error: " + e.localizedMessage + Toast.makeText( + graphicOverlay.context, """ + $error + Cause: ${e.cause} + """.trimIndent(), Toast.LENGTH_SHORT + ).show() + Log.d(TAG, error) + e.printStackTrace() + this@VisionProcessorBase.onFailure(e) + }) + } + + override fun stop() { + executor.shutdown() + isShutdown = true + resetLatencyStats() + fpsTimer.cancel() + } + + private fun resetLatencyStats() { + numRuns = 0 + totalFrameMs = 0 + maxFrameMs = 0 + minFrameMs = Long.MAX_VALUE + totalDetectorMs = 0 + maxDetectorMs = 0 + minDetectorMs = Long.MAX_VALUE + } + + protected abstract fun detectInImage(image: InputImage): Task + + protected open fun detectInImage(image: MlImage): Task { + return Tasks.forException( + MlKitException( + "MlImage is currently not demonstrated for this feature", + MlKitException.INVALID_ARGUMENT + ) + ) + } + + protected abstract fun onSuccess(results: T, graphicOverlay: GraphicOverlay) + + protected abstract fun onFailure(e: Exception) + + protected open fun isMlImageEnabled(context: Context?): Boolean { + return false + } +} diff --git a/app/src/main/java/com/modarb/android/posedetection/classification/ClassificationResult.java b/app/src/main/java/com/modarb/android/posedetection/classification/ClassificationResult.java new file mode 100644 index 0000000..5839bea --- /dev/null +++ b/app/src/main/java/com/modarb/android/posedetection/classification/ClassificationResult.java @@ -0,0 +1,57 @@ +/* + * Copyright 2020 Google LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.modarb.android.posedetection.classification; + +import static java.util.Collections.max; + +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + + +public class ClassificationResult { + + private final Map classConfidences; + + public ClassificationResult() { + classConfidences = new HashMap<>(); + } + + public Set getAllClasses() { + return classConfidences.keySet(); + } + + public float getClassConfidence(String className) { + return classConfidences.containsKey(className) ? classConfidences.get(className) : 0; + } + + public String getMaxConfidenceClass() { + return max( + classConfidences.entrySet(), + (entry1, entry2) -> (int) (entry1.getValue() - entry2.getValue())) + .getKey(); + } + + public void incrementClassConfidence(String className) { + classConfidences.put(className, + classConfidences.containsKey(className) ? classConfidences.get(className) + 1 : 1); + } + + public void putClassConfidence(String className, float confidence) { + classConfidences.put(className, confidence); + } +} diff --git a/app/src/main/java/com/modarb/android/posedetection/classification/EMASmoothing.java b/app/src/main/java/com/modarb/android/posedetection/classification/EMASmoothing.java new file mode 100644 index 0000000..52fca51 --- /dev/null +++ b/app/src/main/java/com/modarb/android/posedetection/classification/EMASmoothing.java @@ -0,0 +1,87 @@ +/* + * Copyright 2020 Google LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.modarb.android.posedetection.classification; + +import android.os.SystemClock; + +import java.util.Deque; +import java.util.HashSet; +import java.util.Set; +import java.util.concurrent.LinkedBlockingDeque; + + +public class EMASmoothing { + private static final int DEFAULT_WINDOW_SIZE = 10; + private static final float DEFAULT_ALPHA = 0.2f; + + private static final long RESET_THRESHOLD_MS = 100; + + private final int windowSize; + private final float alpha; + + private final Deque window; + + private long lastInputMs; + + public EMASmoothing() { + this(DEFAULT_WINDOW_SIZE, DEFAULT_ALPHA); + } + + public EMASmoothing(int windowSize, float alpha) { + this.windowSize = windowSize; + this.alpha = alpha; + this.window = new LinkedBlockingDeque<>(windowSize); + } + + public ClassificationResult getSmoothedResult(ClassificationResult classificationResult) { + long nowMs = SystemClock.elapsedRealtime(); + if (nowMs - lastInputMs > RESET_THRESHOLD_MS) { + window.clear(); + } + lastInputMs = nowMs; + + + if (window.size() == windowSize) { + window.pollLast(); + } + window.addFirst(classificationResult); + + Set allClasses = new HashSet<>(); + for (ClassificationResult result : window) { + allClasses.addAll(result.getAllClasses()); + } + + ClassificationResult smoothedResult = new ClassificationResult(); + + for (String className : allClasses) { + float factor = 1; + float topSum = 0; + float bottomSum = 0; + for (ClassificationResult result : window) { + float value = result.getClassConfidence(className); + + topSum += factor * value; + bottomSum += factor; + + factor = (float) (factor * (1.0 - alpha)); + } + smoothedResult.putClassConfidence(className, topSum / bottomSum); + } + + return smoothedResult; + } +} diff --git a/app/src/main/java/com/modarb/android/posedetection/classification/PoseClassifier.java b/app/src/main/java/com/modarb/android/posedetection/classification/PoseClassifier.java new file mode 100644 index 0000000..3a41336 --- /dev/null +++ b/app/src/main/java/com/modarb/android/posedetection/classification/PoseClassifier.java @@ -0,0 +1,134 @@ +/* + * Copyright 2020 Google LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.modarb.android.posedetection.classification; + +import static com.modarb.android.posedetection.classification.PoseEmbedding.getPoseEmbedding; +import static com.modarb.android.posedetection.classification.Utils.maxAbs; +import static com.modarb.android.posedetection.classification.Utils.multiply; +import static com.modarb.android.posedetection.classification.Utils.multiplyAll; +import static com.modarb.android.posedetection.classification.Utils.subtract; +import static com.modarb.android.posedetection.classification.Utils.sumAbs; +import static java.lang.Math.max; +import static java.lang.Math.min; + +import android.util.Pair; + +import com.google.mlkit.vision.common.PointF3D; +import com.google.mlkit.vision.pose.Pose; +import com.google.mlkit.vision.pose.PoseLandmark; + +import java.util.ArrayList; +import java.util.List; +import java.util.PriorityQueue; + + +public class PoseClassifier { + private static final String TAG = "PoseClassifier"; + private static final int MAX_DISTANCE_TOP_K = 30; + private static final int MEAN_DISTANCE_TOP_K = 10; + private static final PointF3D AXES_WEIGHTS = PointF3D.from(1, 1, 0.2f); + + private final List poseSamples; + private final int maxDistanceTopK; + private final int meanDistanceTopK; + private final PointF3D axesWeights; + + public PoseClassifier(List poseSamples) { + this(poseSamples, MAX_DISTANCE_TOP_K, MEAN_DISTANCE_TOP_K, AXES_WEIGHTS); + } + + public PoseClassifier(List poseSamples, int maxDistanceTopK, int meanDistanceTopK, PointF3D axesWeights) { + this.poseSamples = poseSamples; + this.maxDistanceTopK = maxDistanceTopK; + this.meanDistanceTopK = meanDistanceTopK; + this.axesWeights = axesWeights; + } + + private static List extractPoseLandmarks(Pose pose) { + List landmarks = new ArrayList<>(); + for (PoseLandmark poseLandmark : pose.getAllPoseLandmarks()) { + landmarks.add(poseLandmark.getPosition3D()); + } + return landmarks; + } + + + public int confidenceRange() { + return min(maxDistanceTopK, meanDistanceTopK); + } + + public ClassificationResult classify(Pose pose) { + return classify(extractPoseLandmarks(pose)); + } + + public ClassificationResult classify(List landmarks) { + ClassificationResult result = new ClassificationResult(); + // Return early if no landmarks detected. + if (landmarks.isEmpty()) { + return result; + } + + List flippedLandmarks = new ArrayList<>(landmarks); + multiplyAll(flippedLandmarks, PointF3D.from(-1, 1, 1)); + + List embedding = getPoseEmbedding(landmarks); + List flippedEmbedding = getPoseEmbedding(flippedLandmarks); + + + PriorityQueue> maxDistances = new PriorityQueue<>(maxDistanceTopK, (o1, o2) -> -Float.compare(o1.second, o2.second)); + for (PoseSample poseSample : poseSamples) { + List sampleEmbedding = poseSample.getEmbedding(); + + float originalMax = 0; + float flippedMax = 0; + for (int i = 0; i < embedding.size(); i++) { + originalMax = max(originalMax, maxAbs(multiply(subtract(embedding.get(i), sampleEmbedding.get(i)), axesWeights))); + flippedMax = max(flippedMax, maxAbs(multiply(subtract(flippedEmbedding.get(i), sampleEmbedding.get(i)), axesWeights))); + } + maxDistances.add(new Pair<>(poseSample, min(originalMax, flippedMax))); + + if (maxDistances.size() > maxDistanceTopK) { + maxDistances.poll(); + } + } + + PriorityQueue> meanDistances = new PriorityQueue<>(meanDistanceTopK, (o1, o2) -> -Float.compare(o1.second, o2.second)); + for (Pair sampleDistances : maxDistances) { + PoseSample poseSample = sampleDistances.first; + List sampleEmbedding = poseSample.getEmbedding(); + + float originalSum = 0; + float flippedSum = 0; + for (int i = 0; i < embedding.size(); i++) { + originalSum += sumAbs(multiply(subtract(embedding.get(i), sampleEmbedding.get(i)), axesWeights)); + flippedSum += sumAbs(multiply(subtract(flippedEmbedding.get(i), sampleEmbedding.get(i)), axesWeights)); + } + float meanDistance = min(originalSum, flippedSum) / (embedding.size() * 2); + meanDistances.add(new Pair<>(poseSample, meanDistance)); + if (meanDistances.size() > meanDistanceTopK) { + meanDistances.poll(); + } + } + + for (Pair sampleDistances : meanDistances) { + String className = sampleDistances.first.getClassName(); + result.incrementClassConfidence(className); + } + + return result; + } +} diff --git a/app/src/main/java/com/modarb/android/posedetection/classification/PoseClassifierProcessor.java b/app/src/main/java/com/modarb/android/posedetection/classification/PoseClassifierProcessor.java new file mode 100644 index 0000000..ec2f576 --- /dev/null +++ b/app/src/main/java/com/modarb/android/posedetection/classification/PoseClassifierProcessor.java @@ -0,0 +1,110 @@ + + +package com.modarb.android.posedetection.classification; + +import android.content.Context; +import android.os.Looper; +import android.util.Log; + +import androidx.annotation.WorkerThread; + +import com.google.common.base.Preconditions; +import com.google.mlkit.vision.pose.Pose; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; + + +public class PoseClassifierProcessor { + private static final String TAG = "PoseClassifierProcessor"; + private static final String POSE_SAMPLES_FILE = "pose/fitness_pose_samples.csv"; + + + private static final String PUSHUPS_CLASS = "pushups_down"; + private static final String[] POSE_CLASSES = {PUSHUPS_CLASS}; + + private final boolean isStreamMode; + + private EMASmoothing emaSmoothing; + private List repCounters; + private PoseClassifier poseClassifier; + private String lastRepResult; + + @WorkerThread + public PoseClassifierProcessor(Context context, boolean isStreamMode) { + Preconditions.checkState(Looper.myLooper() != Looper.getMainLooper()); + this.isStreamMode = isStreamMode; + if (isStreamMode) { + emaSmoothing = new EMASmoothing(); + repCounters = new ArrayList<>(); + lastRepResult = ""; + } + loadPoseSamples(context); + } + + private void loadPoseSamples(Context context) { + List poseSamples = new ArrayList<>(); + try { + BufferedReader reader = new BufferedReader(new InputStreamReader(context.getAssets().open(POSE_SAMPLES_FILE))); + String csvLine = reader.readLine(); + while (csvLine != null) { + PoseSample poseSample = PoseSample.getPoseSample(csvLine, ","); + if (poseSample != null) { + poseSamples.add(poseSample); + } + csvLine = reader.readLine(); + } + } catch (IOException e) { + Log.e(TAG, "Error when loading pose samples.\n" + e); + } + poseClassifier = new PoseClassifier(poseSamples); + if (isStreamMode) { + for (String className : POSE_CLASSES) { + repCounters.add(new RepetitionCounter(className)); + } + } + } + + + @WorkerThread + public List getPoseResult(Pose pose) { + Preconditions.checkState(Looper.myLooper() != Looper.getMainLooper()); + List result = new ArrayList<>(); + ClassificationResult classification = poseClassifier.classify(pose); + + if (isStreamMode) { + classification = emaSmoothing.getSmoothedResult(classification); + + if (pose.getAllPoseLandmarks().isEmpty()) { + result.add(lastRepResult); + return result; + } + +// for (RepetitionCounter repCounter : repCounters) { +// int repsBefore = repCounter.getNumRepeats(); +// int repsAfter = repCounter.addClassificationResult(classification); +// if (repsAfter > repsBefore) { +//// ToneGenerator tg = new ToneGenerator(AudioManager.STREAM_NOTIFICATION, 100); +//// tg.startTone(ToneGenerator.TONE_PROP_BEEP); +//// lastRepResult = String.format(Locale.US, "%s : %d reps", repCounter.getClassName(), repsAfter); +// break; +// } +// } + //result.add(lastRepResult); + } + + if (!pose.getAllPoseLandmarks().isEmpty()) { + String maxConfidenceClass = classification.getMaxConfidenceClass(); + if (maxConfidenceClass.contains("push")) { + String maxConfidenceClassResult = String.format(Locale.US, "%s : %.2f confidence", maxConfidenceClass, classification.getClassConfidence(maxConfidenceClass) / poseClassifier.confidenceRange()); + result.add(maxConfidenceClassResult); + } + } + return result; + } + +} diff --git a/app/src/main/java/com/modarb/android/posedetection/classification/PoseEmbedding.java b/app/src/main/java/com/modarb/android/posedetection/classification/PoseEmbedding.java new file mode 100644 index 0000000..697bcf0 --- /dev/null +++ b/app/src/main/java/com/modarb/android/posedetection/classification/PoseEmbedding.java @@ -0,0 +1,131 @@ +/* + * Copyright 2020 Google LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.modarb.android.posedetection.classification; + + +import static com.modarb.android.posedetection.classification.Utils.average; +import static com.modarb.android.posedetection.classification.Utils.l2Norm2D; +import static com.modarb.android.posedetection.classification.Utils.multiplyAll; +import static com.modarb.android.posedetection.classification.Utils.subtract; +import static com.modarb.android.posedetection.classification.Utils.subtractAll; + +import com.google.mlkit.vision.common.PointF3D; +import com.google.mlkit.vision.pose.PoseLandmark; + +import java.util.ArrayList; +import java.util.List; + + +public class PoseEmbedding { + private static final float TORSO_MULTIPLIER = 2.5f; + + private PoseEmbedding() { + } + + public static List getPoseEmbedding(List landmarks) { + List normalizedLandmarks = normalize(landmarks); + return getEmbedding(normalizedLandmarks); + } + + private static List normalize(List landmarks) { + List normalizedLandmarks = new ArrayList<>(landmarks); + PointF3D center = average( + landmarks.get(PoseLandmark.LEFT_HIP), landmarks.get(PoseLandmark.RIGHT_HIP)); + subtractAll(center, normalizedLandmarks); + + multiplyAll(normalizedLandmarks, 1 / getPoseSize(normalizedLandmarks)); + multiplyAll(normalizedLandmarks, 100); + return normalizedLandmarks; + } + + private static float getPoseSize(List landmarks) { + + PointF3D hipsCenter = average( + landmarks.get(PoseLandmark.LEFT_HIP), landmarks.get(PoseLandmark.RIGHT_HIP)); + + PointF3D shouldersCenter = average( + landmarks.get(PoseLandmark.LEFT_SHOULDER), + landmarks.get(PoseLandmark.RIGHT_SHOULDER)); + + float torsoSize = l2Norm2D(subtract(hipsCenter, shouldersCenter)); + + float maxDistance = torsoSize * TORSO_MULTIPLIER; + + for (PointF3D landmark : landmarks) { + float distance = l2Norm2D(subtract(hipsCenter, landmark)); + if (distance > maxDistance) { + maxDistance = distance; + } + } + return maxDistance; + } + + private static List getEmbedding(List lm) { + List embedding = new ArrayList<>(); + + + embedding.add(subtract( + average(lm.get(PoseLandmark.LEFT_HIP), lm.get(PoseLandmark.RIGHT_HIP)), + average(lm.get(PoseLandmark.LEFT_SHOULDER), lm.get(PoseLandmark.RIGHT_SHOULDER)) + )); + + embedding.add(subtract( + lm.get(PoseLandmark.LEFT_SHOULDER), lm.get(PoseLandmark.LEFT_ELBOW))); + embedding.add(subtract( + lm.get(PoseLandmark.RIGHT_SHOULDER), lm.get(PoseLandmark.RIGHT_ELBOW))); + + embedding.add(subtract(lm.get(PoseLandmark.LEFT_ELBOW), lm.get(PoseLandmark.LEFT_WRIST))); + embedding.add(subtract(lm.get(PoseLandmark.RIGHT_ELBOW), lm.get(PoseLandmark.RIGHT_WRIST))); + + embedding.add(subtract(lm.get(PoseLandmark.LEFT_HIP), lm.get(PoseLandmark.LEFT_KNEE))); + embedding.add(subtract(lm.get(PoseLandmark.RIGHT_HIP), lm.get(PoseLandmark.RIGHT_KNEE))); + + embedding.add(subtract(lm.get(PoseLandmark.LEFT_KNEE), lm.get(PoseLandmark.LEFT_ANKLE))); + embedding.add(subtract(lm.get(PoseLandmark.RIGHT_KNEE), lm.get(PoseLandmark.RIGHT_ANKLE))); + + // Two joints. + embedding.add(subtract( + lm.get(PoseLandmark.LEFT_SHOULDER), lm.get(PoseLandmark.LEFT_WRIST))); + embedding.add(subtract( + lm.get(PoseLandmark.RIGHT_SHOULDER), lm.get(PoseLandmark.RIGHT_WRIST))); + + embedding.add(subtract(lm.get(PoseLandmark.LEFT_HIP), lm.get(PoseLandmark.LEFT_ANKLE))); + embedding.add(subtract(lm.get(PoseLandmark.RIGHT_HIP), lm.get(PoseLandmark.RIGHT_ANKLE))); + + // Four joints. + embedding.add(subtract(lm.get(PoseLandmark.LEFT_HIP), lm.get(PoseLandmark.LEFT_WRIST))); + embedding.add(subtract(lm.get(PoseLandmark.RIGHT_HIP), lm.get(PoseLandmark.RIGHT_WRIST))); + + // Five joints. + embedding.add(subtract( + lm.get(PoseLandmark.LEFT_SHOULDER), lm.get(PoseLandmark.LEFT_ANKLE))); + embedding.add(subtract( + lm.get(PoseLandmark.RIGHT_SHOULDER), lm.get(PoseLandmark.RIGHT_ANKLE))); + + embedding.add(subtract(lm.get(PoseLandmark.LEFT_HIP), lm.get(PoseLandmark.LEFT_WRIST))); + embedding.add(subtract(lm.get(PoseLandmark.RIGHT_HIP), lm.get(PoseLandmark.RIGHT_WRIST))); + + // Cross body. + embedding.add(subtract(lm.get(PoseLandmark.LEFT_ELBOW), lm.get(PoseLandmark.RIGHT_ELBOW))); + embedding.add(subtract(lm.get(PoseLandmark.LEFT_KNEE), lm.get(PoseLandmark.RIGHT_KNEE))); + + embedding.add(subtract(lm.get(PoseLandmark.LEFT_WRIST), lm.get(PoseLandmark.RIGHT_WRIST))); + embedding.add(subtract(lm.get(PoseLandmark.LEFT_ANKLE), lm.get(PoseLandmark.RIGHT_ANKLE))); + + return embedding; + } +} diff --git a/app/src/main/java/com/modarb/android/posedetection/classification/PoseSample.java b/app/src/main/java/com/modarb/android/posedetection/classification/PoseSample.java new file mode 100644 index 0000000..420986f --- /dev/null +++ b/app/src/main/java/com/modarb/android/posedetection/classification/PoseSample.java @@ -0,0 +1,80 @@ +/* + * Copyright 2020 Google LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.modarb.android.posedetection.classification; + +import android.util.Log; + +import com.google.common.base.Splitter; +import com.google.mlkit.vision.common.PointF3D; + +import java.util.ArrayList; +import java.util.List; + + +public class PoseSample { + private static final String TAG = "PoseSample"; + private static final int NUM_LANDMARKS = 33; + private static final int NUM_DIMS = 3; + + private final String name; + private final String className; + private final List embedding; + + public PoseSample(String name, String className, List landmarks) { + this.name = name; + this.className = className; + this.embedding = PoseEmbedding.getPoseEmbedding(landmarks); + } + + public static PoseSample getPoseSample(String csvLine, String separator) { + List tokens = Splitter.onPattern(separator).splitToList(csvLine); + + if (tokens.size() != (NUM_LANDMARKS * NUM_DIMS) + 2) { + Log.e(TAG, "Invalid number of tokens for PoseSample"); + return null; + } + String name = tokens.get(0); + String className = tokens.get(1); + List landmarks = new ArrayList<>(); + + for (int i = 2; i < tokens.size(); i += NUM_DIMS) { + try { + landmarks.add( + PointF3D.from( + Float.parseFloat(tokens.get(i)), + Float.parseFloat(tokens.get(i + 1)), + Float.parseFloat(tokens.get(i + 2)))); + } catch (NullPointerException | NumberFormatException e) { + Log.e(TAG, "Invalid value " + tokens.get(i) + " for landmark position."); + return null; + } + } + return new PoseSample(name, className, landmarks); + } + + public String getName() { + return name; + } + + public String getClassName() { + return className; + } + + public List getEmbedding() { + return embedding; + } +} diff --git a/app/src/main/java/com/modarb/android/posedetection/classification/RepetitionCounter.java b/app/src/main/java/com/modarb/android/posedetection/classification/RepetitionCounter.java new file mode 100644 index 0000000..186cb0c --- /dev/null +++ b/app/src/main/java/com/modarb/android/posedetection/classification/RepetitionCounter.java @@ -0,0 +1,68 @@ +/* + * Copyright 2020 Google LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.modarb.android.posedetection.classification; + + +public class RepetitionCounter { + + private static final float DEFAULT_ENTER_THRESHOLD = 6f; + private static final float DEFAULT_EXIT_THRESHOLD = 4f; + + private final String className; + private final float enterThreshold; + private final float exitThreshold; + + private int numRepeats; + private boolean poseEntered; + + public RepetitionCounter(String className) { + this(className, DEFAULT_ENTER_THRESHOLD, DEFAULT_EXIT_THRESHOLD); + } + + public RepetitionCounter(String className, float enterThreshold, float exitThreshold) { + this.className = className; + this.enterThreshold = enterThreshold; + this.exitThreshold = exitThreshold; + numRepeats = 0; + poseEntered = false; + } + + + public int addClassificationResult(ClassificationResult classificationResult) { + float poseConfidence = classificationResult.getClassConfidence(className); + + if (!poseEntered) { + poseEntered = poseConfidence > enterThreshold; + return numRepeats; + } + + if (poseConfidence < exitThreshold) { + numRepeats++; + poseEntered = false; + } + + return numRepeats; + } + + public String getClassName() { + return className; + } + + public int getNumRepeats() { + return numRepeats; + } +} diff --git a/app/src/main/java/com/modarb/android/posedetection/classification/Utils.java b/app/src/main/java/com/modarb/android/posedetection/classification/Utils.java new file mode 100644 index 0000000..00ca0c5 --- /dev/null +++ b/app/src/main/java/com/modarb/android/posedetection/classification/Utils.java @@ -0,0 +1,91 @@ +/* + * Copyright 2020 Google LLC. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.modarb.android.posedetection.classification; + +import static com.google.common.primitives.Floats.max; + +import com.google.mlkit.vision.common.PointF3D; + +import java.util.List; +import java.util.ListIterator; + +public class Utils { + private Utils() { + } + + public static PointF3D add(PointF3D a, PointF3D b) { + return PointF3D.from(a.getX() + b.getX(), a.getY() + b.getY(), a.getZ() + b.getZ()); + } + + public static PointF3D subtract(PointF3D b, PointF3D a) { + return PointF3D.from(a.getX() - b.getX(), a.getY() - b.getY(), a.getZ() - b.getZ()); + } + + public static PointF3D multiply(PointF3D a, float multiple) { + return PointF3D.from(a.getX() * multiple, a.getY() * multiple, a.getZ() * multiple); + } + + public static PointF3D multiply(PointF3D a, PointF3D multiple) { + return PointF3D.from( + a.getX() * multiple.getX(), a.getY() * multiple.getY(), a.getZ() * multiple.getZ()); + } + + public static PointF3D average(PointF3D a, PointF3D b) { + return PointF3D.from( + (a.getX() + b.getX()) * 0.5f, (a.getY() + b.getY()) * 0.5f, (a.getZ() + b.getZ()) * 0.5f); + } + + public static float l2Norm2D(PointF3D point) { + return (float) Math.hypot(point.getX(), point.getY()); + } + + public static float maxAbs(PointF3D point) { + return max(Math.abs(point.getX()), Math.abs(point.getY()), Math.abs(point.getZ())); + } + + public static float sumAbs(PointF3D point) { + return Math.abs(point.getX()) + Math.abs(point.getY()) + Math.abs(point.getZ()); + } + + public static void addAll(List pointsList, PointF3D p) { + ListIterator iterator = pointsList.listIterator(); + while (iterator.hasNext()) { + iterator.set(add(iterator.next(), p)); + } + } + + public static void subtractAll(PointF3D p, List pointsList) { + ListIterator iterator = pointsList.listIterator(); + while (iterator.hasNext()) { + iterator.set(subtract(p, iterator.next())); + } + } + + public static void multiplyAll(List pointsList, float multiple) { + ListIterator iterator = pointsList.listIterator(); + while (iterator.hasNext()) { + iterator.set(multiply(iterator.next(), multiple)); + } + } + + public static void multiplyAll(List pointsList, PointF3D multiple) { + ListIterator iterator = pointsList.listIterator(); + while (iterator.hasNext()) { + iterator.set(multiply(iterator.next(), multiple)); + } + } +} diff --git a/app/src/main/java/com/modarb/android/posedetection/posedetector/PoseDetectorProcessor.kt b/app/src/main/java/com/modarb/android/posedetection/posedetector/PoseDetectorProcessor.kt new file mode 100644 index 0000000..3dd090d --- /dev/null +++ b/app/src/main/java/com/modarb/android/posedetection/posedetector/PoseDetectorProcessor.kt @@ -0,0 +1,361 @@ +package com.modarb.android.posedetection.posedetector + +import android.content.Context +import android.os.Build +import android.speech.tts.TextToSpeech +import android.util.Log +import androidx.annotation.RequiresApi +import com.google.android.gms.tasks.Task +import com.google.android.odml.image.MlImage +import com.google.mlkit.vision.common.InputImage +import com.google.mlkit.vision.demo.kotlin.VisionProcessorBase +import com.google.mlkit.vision.pose.Pose +import com.google.mlkit.vision.pose.PoseDetection +import com.google.mlkit.vision.pose.PoseDetector +import com.google.mlkit.vision.pose.PoseDetectorOptionsBase +import com.google.mlkit.vision.pose.PoseLandmark +import com.modarb.android.posedetection.GraphicOverlay +import com.modarb.android.posedetection.classification.PoseClassifierProcessor +import kotlinx.coroutines.CoroutineExceptionHandler +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.delay +import kotlinx.coroutines.isActive +import kotlinx.coroutines.launch +import java.util.Locale +import java.util.concurrent.Executor +import java.util.concurrent.Executors +import kotlin.coroutines.CoroutineContext +import kotlin.math.atan2 + +class PoseDetectorProcessor( + private val context: Context, + options: PoseDetectorOptionsBase, + private val showInFrameLikelihood: Boolean, + private val visualizeZ: Boolean, + private val rescaleZForVisualization: Boolean, + private val runClassification: Boolean, + private val isStreamMode: Boolean +) : VisionProcessorBase(context), CoroutineScope { + + private val detector: PoseDetector + private val classificationExecutor: Executor + + private var poseClassifierProcessor: PoseClassifierProcessor? = null + + class PoseWithClassification(val pose: Pose, val classificationResult: List) + + init { + detector = PoseDetection.getClient(options) + classificationExecutor = Executors.newSingleThreadExecutor() + } + + override fun stop() { + super.stop() + detector.close() + stopPushUpFormCheck() + } + + override fun detectInImage(image: InputImage): Task { + return detector.process(image).continueWith(classificationExecutor) { task -> + val pose = task.getResult() + var classificationResult: List = ArrayList() + if (runClassification) { + if (poseClassifierProcessor == null) { + poseClassifierProcessor = PoseClassifierProcessor(context, isStreamMode) + } + classificationResult = poseClassifierProcessor!!.getPoseResult(pose) + } + PoseWithClassification(pose, classificationResult) + } + } + + + override fun detectInImage(image: MlImage): Task { + return detector.process(image).continueWith( + classificationExecutor + ) { task -> + val pose = task.getResult() + var classificationResult: List = ArrayList() + if (runClassification) { + if (poseClassifierProcessor == null) { + poseClassifierProcessor = PoseClassifierProcessor(context, isStreamMode) + } + classificationResult = poseClassifierProcessor!!.getPoseResult(pose) + } + PoseWithClassification(pose, classificationResult) + } + } + + private var pushUpCheckJob: Job? = null + + override fun onSuccess( + results: PoseWithClassification, graphicOverlay: GraphicOverlay + ) { + graphicOverlay.add( + PoseGraphic( + graphicOverlay, + results.pose, + showInFrameLikelihood, + visualizeZ, + rescaleZForVisualization, + results.classificationResult + ) + ) + Log.d("Result", results.classificationResult.toString()) + + val containsPushup = results.classificationResult.any { "pushup" in it } + + if (containsPushup) { + startPushUpFormCheck(results) + } else { + stopPushUpFormCheck() + } + } + + private val coroutineExceptionHandler = CoroutineExceptionHandler { _, throwable -> + println("Coroutine exception occurred: ${throwable.message}") + } + + override val coroutineContext: CoroutineContext + get() = Dispatchers.Main + coroutineExceptionHandler + + private fun startPushUpFormCheck(results: PoseWithClassification) { + pushUpCheckJob?.cancel() + pushUpCheckJob = launch { + while (isActive) { + results.pose.allPoseLandmarks.let { + checkPushUpForm(it) + } + delay(1000) + } + } + } + + private fun stopPushUpFormCheck() { + pushUpCheckJob?.cancel() + } + + override fun onFailure(e: Exception) { + Log.e(TAG, "Pose detection failed!", e) + } + + override fun isMlImageEnabled(context: Context?): Boolean { + return true + } + + + private fun debugPose(pose: Pose) { + val landmarkTypes = listOf( + PoseLandmark.NOSE, + PoseLandmark.LEFT_EYE_INNER, + PoseLandmark.LEFT_EYE, + PoseLandmark.LEFT_EYE_OUTER, + PoseLandmark.RIGHT_EYE_INNER, + PoseLandmark.RIGHT_EYE, + PoseLandmark.RIGHT_EYE_OUTER, + PoseLandmark.LEFT_EAR, + PoseLandmark.RIGHT_EAR, + PoseLandmark.LEFT_SHOULDER, + PoseLandmark.RIGHT_SHOULDER, + PoseLandmark.LEFT_ELBOW, + PoseLandmark.RIGHT_ELBOW, + PoseLandmark.LEFT_WRIST, + PoseLandmark.RIGHT_WRIST, + PoseLandmark.LEFT_HIP, + PoseLandmark.RIGHT_HIP, + PoseLandmark.LEFT_KNEE, + PoseLandmark.RIGHT_KNEE, + PoseLandmark.LEFT_ANKLE, + PoseLandmark.RIGHT_ANKLE + ) + + for (landmarkType in landmarkTypes) { + val landmark = pose.getPoseLandmark(landmarkType) + if (landmark != null) { + Log.d( + TAG, + "Landmark: ${landmark.landmarkType}, Position: ${landmark.position3D.x}, ${landmark.position3D.y}, ${landmark.position3D.z}" + ) + } else { + Log.d(TAG, "Landmark: $landmarkType, Not detected") + } + } + + } + + private fun getAngle( + firstPoint: PoseLandmark, midPoint: PoseLandmark, lastPoint: PoseLandmark + ): Double { + var result = Math.toDegrees( + (atan2( + lastPoint.position.y - midPoint.position.y, + lastPoint.position.x - midPoint.position.x + ) - atan2( + firstPoint.position.y - midPoint.position.y, + firstPoint.position.x - midPoint.position.x + )).toDouble() + ) + result = Math.abs(result) + if (result > 180) { + result = 360.0 - result + } + return result + } + + + private var textToSpeech: TextToSpeech? = null + private var isSpeaking: Boolean = false + + init { + textToSpeech = TextToSpeech(context) { status -> + if (status == TextToSpeech.SUCCESS) { + textToSpeech?.language = Locale.US + textToSpeech?.setOnUtteranceCompletedListener { + isSpeaking = false + } + } else { + + } + } + } + + @RequiresApi(Build.VERSION_CODES.LOLLIPOP) + fun isBodyStraight(landmarks: List): Pair { + val leftShoulder = + landmarks.find { it.landmarkType == PoseLandmark.LEFT_SHOULDER } ?: return Pair( + false, "Left shoulder not found" + ) + val rightShoulder = + landmarks.find { it.landmarkType == PoseLandmark.RIGHT_SHOULDER } ?: return Pair( + false, "Right shoulder not found" + ) + val leftHip = landmarks.find { it.landmarkType == PoseLandmark.LEFT_HIP } ?: return Pair( + false, "Left hip not found" + ) + val rightHip = landmarks.find { it.landmarkType == PoseLandmark.RIGHT_HIP } ?: return Pair( + false, "Right hip not found" + ) + val leftKnee = landmarks.find { it.landmarkType == PoseLandmark.LEFT_KNEE } ?: return Pair( + false, "Left knee not found" + ) + val rightKnee = + landmarks.find { it.landmarkType == PoseLandmark.RIGHT_KNEE } ?: return Pair( + false, "Right knee not found" + ) + + val leftBodyAngle = getAngle(leftShoulder, leftHip, leftKnee) + val rightBodyAngle = getAngle(rightShoulder, rightHip, rightKnee) + + return if (leftBodyAngle > 160 && rightBodyAngle > 160) { + Pair(true, "Body is straight") + } else { + speak("Straight your body.") + Pair( + false, + "Body is not straight. Left body angle: $leftBodyAngle, right body angle: $rightBodyAngle" + ) + } + } + + private fun speak(text: String) { + if (!isSpeaking) { + isSpeaking = true + val params = HashMap() + params[TextToSpeech.Engine.KEY_PARAM_UTTERANCE_ID] = "uniqueId" + textToSpeech?.speak(text, TextToSpeech.QUEUE_FLUSH, params) + } + } + + fun shutdown() { + textToSpeech?.shutdown() + } + + + private fun areShouldersAboveWrists(landmarks: List): Pair { + val leftShoulder = + landmarks.find { it.landmarkType == PoseLandmark.LEFT_SHOULDER } ?: return Pair( + false, "Left shoulder not found" + ) + val rightShoulder = + landmarks.find { it.landmarkType == PoseLandmark.RIGHT_SHOULDER } ?: return Pair( + false, "Right shoulder not found" + ) + val leftWrist = + landmarks.find { it.landmarkType == PoseLandmark.LEFT_WRIST } ?: return Pair( + false, "Left wrist not found" + ) + val rightWrist = + landmarks.find { it.landmarkType == PoseLandmark.RIGHT_WRIST } ?: return Pair( + false, "Right wrist not found" + ) + + return if (leftShoulder.position.y < leftWrist.position.y && rightShoulder.position.y < rightWrist.position.y) { + Pair(true, "Shoulders are above wrists") + } else { + speak("Shoulders are not above wrists.") + Pair( + false, + "Shoulders are not above wrists. Left shoulder y: ${leftShoulder.position.y}, left wrist y: ${leftWrist.position.y}, right shoulder y: ${rightShoulder.position.y}, right wrist y: ${rightWrist.position.y}" + ) + } + } + + private fun areElbowsBendingCorrectly(landmarks: List): Pair { + val leftElbow = + landmarks.find { it.landmarkType == PoseLandmark.LEFT_ELBOW } ?: return Pair( + false, "Left elbow not found" + ) + val rightElbow = + landmarks.find { it.landmarkType == PoseLandmark.RIGHT_ELBOW } ?: return Pair( + false, "Right elbow not found" + ) + val leftShoulder = + landmarks.find { it.landmarkType == PoseLandmark.LEFT_SHOULDER } ?: return Pair( + false, "Left shoulder not found" + ) + val rightShoulder = + landmarks.find { it.landmarkType == PoseLandmark.RIGHT_SHOULDER } ?: return Pair( + false, "Right shoulder not found" + ) + val leftWrist = + landmarks.find { it.landmarkType == PoseLandmark.LEFT_WRIST } ?: return Pair( + false, "Left wrist not found" + ) + val rightWrist = + landmarks.find { it.landmarkType == PoseLandmark.RIGHT_WRIST } ?: return Pair( + false, "Right wrist not found" + ) + + val leftElbowAngle = getAngle(leftShoulder, leftElbow, leftWrist) + val rightElbowAngle = getAngle(rightShoulder, rightElbow, rightWrist) + + return if (leftElbowAngle < 170 && rightElbowAngle < 170) { + Pair(true, "Elbows are bending correctly") + } else { + //speak("Elbows are not bending correctly.") + Pair( + false, + "Elbows are not bending correctly. Left elbow angle: $leftElbowAngle, right elbow angle: $rightElbowAngle" + ) + } + } + + private fun checkPushUpForm(landmarks: List) { + val (_, shouldersAboveWristsMsg) = areShouldersAboveWrists(landmarks) + val (_, bodyStraightMsg) = isBodyStraight(landmarks) + val (_, elbowsBendingCorrectlyMsg) = areElbowsBendingCorrectly( + landmarks + ) + + println(shouldersAboveWristsMsg) + println(bodyStraightMsg) + println(elbowsBendingCorrectlyMsg) + } + + + companion object { + private val TAG = "PoseDetectorProcessor" + } +} diff --git a/app/src/main/java/com/modarb/android/posedetection/posedetector/PoseGraphic.kt b/app/src/main/java/com/modarb/android/posedetection/posedetector/PoseGraphic.kt new file mode 100644 index 0000000..a3c5839 --- /dev/null +++ b/app/src/main/java/com/modarb/android/posedetection/posedetector/PoseGraphic.kt @@ -0,0 +1,217 @@ +package com.modarb.android.posedetection.posedetector + +import android.graphics.Canvas +import android.graphics.Color +import android.graphics.Paint + +import com.google.mlkit.vision.pose.Pose +import com.google.mlkit.vision.pose.PoseLandmark +import com.modarb.android.posedetection.GraphicOverlay +import java.lang.Math.max +import java.lang.Math.min +import java.util.Locale + +class PoseGraphic +internal constructor( + overlay: GraphicOverlay, + private val pose: Pose, + private val showInFrameLikelihood: Boolean, + private val visualizeZ: Boolean, + private val rescaleZForVisualization: Boolean, + private val poseClassification: List +) : GraphicOverlay.Graphic(overlay) { + private var zMin = java.lang.Float.MAX_VALUE + private var zMax = java.lang.Float.MIN_VALUE + private val classificationTextPaint: Paint + private val leftPaint: Paint + private val rightPaint: Paint + private val whitePaint: Paint + + init { + classificationTextPaint = Paint() + classificationTextPaint.color = Color.WHITE + classificationTextPaint.textSize = POSE_CLASSIFICATION_TEXT_SIZE + classificationTextPaint.setShadowLayer(5.0f, 0f, 0f, Color.BLACK) + + whitePaint = Paint() + whitePaint.strokeWidth = STROKE_WIDTH + whitePaint.color = Color.WHITE + whitePaint.textSize = IN_FRAME_LIKELIHOOD_TEXT_SIZE + leftPaint = Paint() + leftPaint.strokeWidth = STROKE_WIDTH + leftPaint.color = Color.GREEN + rightPaint = Paint() + rightPaint.strokeWidth = STROKE_WIDTH + rightPaint.color = Color.YELLOW + } + + override fun draw(canvas: Canvas) { + val landmarks = pose.allPoseLandmarks + if (landmarks.isEmpty()) { + return + } + + // Draw pose classification text. + val classificationX = POSE_CLASSIFICATION_TEXT_SIZE * 0.5f + for (i in poseClassification.indices) { + val classificationY = + canvas.height - + (POSE_CLASSIFICATION_TEXT_SIZE * 1.5f * (poseClassification.size - i).toFloat()) + canvas.drawText( + poseClassification[i], + classificationX, + classificationY, + classificationTextPaint + ) + } + + // Draw all the points + for (landmark in landmarks) { + drawPoint(canvas, landmark, whitePaint) + if (visualizeZ && rescaleZForVisualization) { + zMin = min(zMin, landmark.position3D.z) + zMax = max(zMax, landmark.position3D.z) + } + } + + val nose = pose.getPoseLandmark(PoseLandmark.NOSE) + val lefyEyeInner = pose.getPoseLandmark(PoseLandmark.LEFT_EYE_INNER) + val lefyEye = pose.getPoseLandmark(PoseLandmark.LEFT_EYE) + val leftEyeOuter = pose.getPoseLandmark(PoseLandmark.LEFT_EYE_OUTER) + val rightEyeInner = pose.getPoseLandmark(PoseLandmark.RIGHT_EYE_INNER) + val rightEye = pose.getPoseLandmark(PoseLandmark.RIGHT_EYE) + val rightEyeOuter = pose.getPoseLandmark(PoseLandmark.RIGHT_EYE_OUTER) + val leftEar = pose.getPoseLandmark(PoseLandmark.LEFT_EAR) + val rightEar = pose.getPoseLandmark(PoseLandmark.RIGHT_EAR) + val leftMouth = pose.getPoseLandmark(PoseLandmark.LEFT_MOUTH) + val rightMouth = pose.getPoseLandmark(PoseLandmark.RIGHT_MOUTH) + + val leftShoulder = pose.getPoseLandmark(PoseLandmark.LEFT_SHOULDER) + val rightShoulder = pose.getPoseLandmark(PoseLandmark.RIGHT_SHOULDER) + val leftElbow = pose.getPoseLandmark(PoseLandmark.LEFT_ELBOW) + val rightElbow = pose.getPoseLandmark(PoseLandmark.RIGHT_ELBOW) + val leftWrist = pose.getPoseLandmark(PoseLandmark.LEFT_WRIST) + val rightWrist = pose.getPoseLandmark(PoseLandmark.RIGHT_WRIST) + val leftHip = pose.getPoseLandmark(PoseLandmark.LEFT_HIP) + val rightHip = pose.getPoseLandmark(PoseLandmark.RIGHT_HIP) + val leftKnee = pose.getPoseLandmark(PoseLandmark.LEFT_KNEE) + val rightKnee = pose.getPoseLandmark(PoseLandmark.RIGHT_KNEE) + val leftAnkle = pose.getPoseLandmark(PoseLandmark.LEFT_ANKLE) + val rightAnkle = pose.getPoseLandmark(PoseLandmark.RIGHT_ANKLE) + + val leftPinky = pose.getPoseLandmark(PoseLandmark.LEFT_PINKY) + val rightPinky = pose.getPoseLandmark(PoseLandmark.RIGHT_PINKY) + val leftIndex = pose.getPoseLandmark(PoseLandmark.LEFT_INDEX) + val rightIndex = pose.getPoseLandmark(PoseLandmark.RIGHT_INDEX) + val leftThumb = pose.getPoseLandmark(PoseLandmark.LEFT_THUMB) + val rightThumb = pose.getPoseLandmark(PoseLandmark.RIGHT_THUMB) + val leftHeel = pose.getPoseLandmark(PoseLandmark.LEFT_HEEL) + val rightHeel = pose.getPoseLandmark(PoseLandmark.RIGHT_HEEL) + val leftFootIndex = pose.getPoseLandmark(PoseLandmark.LEFT_FOOT_INDEX) + val rightFootIndex = pose.getPoseLandmark(PoseLandmark.RIGHT_FOOT_INDEX) + + // Face + drawLine(canvas, nose, lefyEyeInner, whitePaint) + drawLine(canvas, lefyEyeInner, lefyEye, whitePaint) + drawLine(canvas, lefyEye, leftEyeOuter, whitePaint) + drawLine(canvas, leftEyeOuter, leftEar, whitePaint) + drawLine(canvas, nose, rightEyeInner, whitePaint) + drawLine(canvas, rightEyeInner, rightEye, whitePaint) + drawLine(canvas, rightEye, rightEyeOuter, whitePaint) + drawLine(canvas, rightEyeOuter, rightEar, whitePaint) + drawLine(canvas, leftMouth, rightMouth, whitePaint) + + drawLine(canvas, leftShoulder, rightShoulder, whitePaint) + drawLine(canvas, leftHip, rightHip, whitePaint) + + // Left body + drawLine(canvas, leftShoulder, leftElbow, leftPaint) + drawLine(canvas, leftElbow, leftWrist, leftPaint) + drawLine(canvas, leftShoulder, leftHip, leftPaint) + drawLine(canvas, leftHip, leftKnee, leftPaint) + drawLine(canvas, leftKnee, leftAnkle, leftPaint) + drawLine(canvas, leftWrist, leftThumb, leftPaint) + drawLine(canvas, leftWrist, leftPinky, leftPaint) + drawLine(canvas, leftWrist, leftIndex, leftPaint) + drawLine(canvas, leftIndex, leftPinky, leftPaint) + drawLine(canvas, leftAnkle, leftHeel, leftPaint) + drawLine(canvas, leftHeel, leftFootIndex, leftPaint) + + // Right body + drawLine(canvas, rightShoulder, rightElbow, rightPaint) + drawLine(canvas, rightElbow, rightWrist, rightPaint) + drawLine(canvas, rightShoulder, rightHip, rightPaint) + drawLine(canvas, rightHip, rightKnee, rightPaint) + drawLine(canvas, rightKnee, rightAnkle, rightPaint) + drawLine(canvas, rightWrist, rightThumb, rightPaint) + drawLine(canvas, rightWrist, rightPinky, rightPaint) + drawLine(canvas, rightWrist, rightIndex, rightPaint) + drawLine(canvas, rightIndex, rightPinky, rightPaint) + drawLine(canvas, rightAnkle, rightHeel, rightPaint) + drawLine(canvas, rightHeel, rightFootIndex, rightPaint) + + // Draw inFrameLikelihood for all points + if (showInFrameLikelihood) { + for (landmark in landmarks) { + canvas.drawText( + String.format(Locale.US, "%.2f", landmark.inFrameLikelihood), + translateX(landmark.position.x), + translateY(landmark.position.y), + whitePaint + ) + } + } + } + + internal fun drawPoint(canvas: Canvas, landmark: PoseLandmark, paint: Paint) { + val point = landmark.position3D + updatePaintColorByZValue( + paint, + canvas, + visualizeZ, + rescaleZForVisualization, + point.z, + zMin, + zMax + ) + canvas.drawCircle(translateX(point.x), translateY(point.y), DOT_RADIUS, paint) + } + + internal fun drawLine( + canvas: Canvas, + startLandmark: PoseLandmark?, + endLandmark: PoseLandmark?, + paint: Paint + ) { + val start = startLandmark!!.position3D + val end = endLandmark!!.position3D + + // Gets average z for the current body line + val avgZInImagePixel = (start.z + end.z) / 2 + updatePaintColorByZValue( + paint, + canvas, + visualizeZ, + rescaleZForVisualization, + avgZInImagePixel, + zMin, + zMax + ) + + canvas.drawLine( + translateX(start.x), + translateY(start.y), + translateX(end.x), + translateY(end.y), + paint + ) + } + + companion object { + + private val DOT_RADIUS = 8.0f + private val IN_FRAME_LIKELIHOOD_TEXT_SIZE = 30.0f + private val STROKE_WIDTH = 10.0f + private val POSE_CLASSIFICATION_TEXT_SIZE = 60.0f + } +} diff --git a/app/src/main/java/com/modarb/android/ui/helpers/ViewUtils.kt b/app/src/main/java/com/modarb/android/ui/helpers/ViewUtils.kt index b74df6f..99997aa 100644 --- a/app/src/main/java/com/modarb/android/ui/helpers/ViewUtils.kt +++ b/app/src/main/java/com/modarb/android/ui/helpers/ViewUtils.kt @@ -9,12 +9,8 @@ object ViewUtils { fun loadImage(context: Context, imageUrl: String, imageView: ImageView) { - Glide - .with(context) - .load(imageUrl) - .placeholder(R.drawable.baseline_broken_image_24) - .centerCrop() - .into(imageView) + Glide.with(context).load(imageUrl).placeholder(R.drawable.baseline_broken_image_24) + .centerCrop().into(imageView) } diff --git a/app/src/main/java/com/modarb/android/ui/home/ui/home/HomeFragment.kt b/app/src/main/java/com/modarb/android/ui/home/ui/home/HomeFragment.kt index c8d146e..74bba06 100644 --- a/app/src/main/java/com/modarb/android/ui/home/ui/home/HomeFragment.kt +++ b/app/src/main/java/com/modarb/android/ui/home/ui/home/HomeFragment.kt @@ -19,6 +19,7 @@ import com.modarb.android.R import com.modarb.android.databinding.FragmentHomeBinding import com.modarb.android.network.ApiResult import com.modarb.android.network.NetworkHelper +import com.modarb.android.posedetection.RequestPermissionsActivity import com.modarb.android.ui.ChatBotWebView import com.modarb.android.ui.helpers.WorkoutData import com.modarb.android.ui.home.HomeActivity @@ -247,6 +248,9 @@ class HomeFragment : Fragment() { startActivity(Intent(requireContext(), ChatBotWebView::class.java)) } + binding.cameraBtn.setOnClickListener { + startActivity(Intent(requireContext(), RequestPermissionsActivity::class.java)) + } binding.userName.text = "Hey, \n" + UserPrefUtil.getUserData(requireContext())!!.user.name } diff --git a/app/src/main/java/com/modarb/android/ui/onboarding/activities/WelcomeScreenActivity.kt b/app/src/main/java/com/modarb/android/ui/onboarding/activities/WelcomeScreenActivity.kt index bac47ff..b44e79f 100644 --- a/app/src/main/java/com/modarb/android/ui/onboarding/activities/WelcomeScreenActivity.kt +++ b/app/src/main/java/com/modarb/android/ui/onboarding/activities/WelcomeScreenActivity.kt @@ -1,9 +1,11 @@ package com.modarb.android.ui.onboarding.activities +import android.app.AlertDialog import android.content.Intent import android.os.Bundle import android.view.View import android.widget.Button +import android.widget.EditText import android.widget.ProgressBar import android.widget.Toast import androidx.appcompat.app.AppCompatActivity @@ -36,10 +38,35 @@ class WelcomeScreenActivity : AppCompatActivity() { val view = binding.root setContentView(view) init() + initServerDialog() initViewModels() onRegister() } + private fun initServerDialog() { + binding.titleTv.setOnClickListener { + val builder = AlertDialog.Builder(this) + builder.setTitle("Change Server Link") + + val input = EditText(this) + input.hint = "Enter new server link" + input.setText(RetrofitService.BASE_URL) + builder.setView(input) + builder.setPositiveButton("OK") { dialog, which -> + val newUrl = input.text.toString() + if (newUrl.isNotEmpty()) { + RetrofitService.changeBaseUrl(newUrl) + var i = Intent(this, WelcomeScreenActivity::class.java) + startActivity(i) + finish() + } + } + builder.setNegativeButton("Cancel") { dialog, which -> dialog.cancel() } + + builder.show() + } + } + private fun init() { initBottomSheet() diff --git a/app/src/main/res/drawable/baseline_cameraswitch_24.xml b/app/src/main/res/drawable/baseline_cameraswitch_24.xml new file mode 100644 index 0000000..e58967f --- /dev/null +++ b/app/src/main/res/drawable/baseline_cameraswitch_24.xml @@ -0,0 +1,20 @@ + + + + + + + + + diff --git a/app/src/main/res/drawable/ic_launcher_background.xml b/app/src/main/res/drawable/ic_launcher_background.xml index 07d5da9..08f3f13 100644 --- a/app/src/main/res/drawable/ic_launcher_background.xml +++ b/app/src/main/res/drawable/ic_launcher_background.xml @@ -1,7 +1,8 @@ - + + + diff --git a/app/src/main/res/layout/activity_app_appearance.xml b/app/src/main/res/layout/activity_app_appearance.xml index 8d92eec..a2abdfb 100644 --- a/app/src/main/res/layout/activity_app_appearance.xml +++ b/app/src/main/res/layout/activity_app_appearance.xml @@ -131,7 +131,7 @@ android:layout_height="wrap_content"> diff --git a/app/src/main/res/layout/activity_camera_setting.xml b/app/src/main/res/layout/activity_camera_setting.xml new file mode 100644 index 0000000..9c37b46 --- /dev/null +++ b/app/src/main/res/layout/activity_camera_setting.xml @@ -0,0 +1,8 @@ + + + + diff --git a/app/src/main/res/layout/activity_camera_view.xml b/app/src/main/res/layout/activity_camera_view.xml new file mode 100644 index 0000000..081d9c7 --- /dev/null +++ b/app/src/main/res/layout/activity_camera_view.xml @@ -0,0 +1,64 @@ + + + + + + + + + + + + + + + + + diff --git a/app/src/main/res/layout/activity_injury_today_workout.xml b/app/src/main/res/layout/activity_injury_today_workout.xml index 0f7cd4f..51004b3 100644 --- a/app/src/main/res/layout/activity_injury_today_workout.xml +++ b/app/src/main/res/layout/activity_injury_today_workout.xml @@ -23,7 +23,7 @@ tools:ignore="ContentDescription" /> diff --git a/app/src/main/res/layout/activity_request_permissions.xml b/app/src/main/res/layout/activity_request_permissions.xml new file mode 100644 index 0000000..984fadc --- /dev/null +++ b/app/src/main/res/layout/activity_request_permissions.xml @@ -0,0 +1,7 @@ + + + + \ No newline at end of file diff --git a/app/src/main/res/layout/activity_welcome_screen.xml b/app/src/main/res/layout/activity_welcome_screen.xml index bf29408..4841fc7 100644 --- a/app/src/main/res/layout/activity_welcome_screen.xml +++ b/app/src/main/res/layout/activity_welcome_screen.xml @@ -40,7 +40,7 @@ app:layout_constraintVertical_bias="0.445" /> + diff --git a/app/src/main/res/mipmap-anydpi-v26/ic_launcher.xml b/app/src/main/res/mipmap-anydpi-v26/ic_launcher.xml index 6f3b755..00f9eaa 100644 --- a/app/src/main/res/mipmap-anydpi-v26/ic_launcher.xml +++ b/app/src/main/res/mipmap-anydpi-v26/ic_launcher.xml @@ -1,6 +1,5 @@ - - + \ No newline at end of file diff --git a/app/src/main/res/mipmap-anydpi-v26/ic_launcher_round.xml b/app/src/main/res/mipmap-anydpi-v26/ic_launcher_round.xml index 6f3b755..00f9eaa 100644 --- a/app/src/main/res/mipmap-anydpi-v26/ic_launcher_round.xml +++ b/app/src/main/res/mipmap-anydpi-v26/ic_launcher_round.xml @@ -1,6 +1,5 @@ - - + \ No newline at end of file diff --git a/app/src/main/res/mipmap-hdpi/ic_launcher.webp b/app/src/main/res/mipmap-hdpi/ic_launcher.webp index c209e78..5236603 100644 Binary files a/app/src/main/res/mipmap-hdpi/ic_launcher.webp and b/app/src/main/res/mipmap-hdpi/ic_launcher.webp differ diff --git a/app/src/main/res/mipmap-hdpi/ic_launcher_foreground.webp b/app/src/main/res/mipmap-hdpi/ic_launcher_foreground.webp new file mode 100644 index 0000000..c75edae Binary files /dev/null and b/app/src/main/res/mipmap-hdpi/ic_launcher_foreground.webp differ diff --git a/app/src/main/res/mipmap-hdpi/ic_launcher_round.webp b/app/src/main/res/mipmap-hdpi/ic_launcher_round.webp index b2dfe3d..6901bf7 100644 Binary files a/app/src/main/res/mipmap-hdpi/ic_launcher_round.webp and b/app/src/main/res/mipmap-hdpi/ic_launcher_round.webp differ diff --git a/app/src/main/res/mipmap-mdpi/ic_launcher.webp b/app/src/main/res/mipmap-mdpi/ic_launcher.webp index 4f0f1d6..0c9a13b 100644 Binary files a/app/src/main/res/mipmap-mdpi/ic_launcher.webp and b/app/src/main/res/mipmap-mdpi/ic_launcher.webp differ diff --git a/app/src/main/res/mipmap-mdpi/ic_launcher_foreground.webp b/app/src/main/res/mipmap-mdpi/ic_launcher_foreground.webp new file mode 100644 index 0000000..c89abc8 Binary files /dev/null and b/app/src/main/res/mipmap-mdpi/ic_launcher_foreground.webp differ diff --git a/app/src/main/res/mipmap-mdpi/ic_launcher_round.webp b/app/src/main/res/mipmap-mdpi/ic_launcher_round.webp index 62b611d..955c983 100644 Binary files a/app/src/main/res/mipmap-mdpi/ic_launcher_round.webp and b/app/src/main/res/mipmap-mdpi/ic_launcher_round.webp differ diff --git a/app/src/main/res/mipmap-xhdpi/ic_launcher.webp b/app/src/main/res/mipmap-xhdpi/ic_launcher.webp index 948a307..d39afd7 100644 Binary files a/app/src/main/res/mipmap-xhdpi/ic_launcher.webp and b/app/src/main/res/mipmap-xhdpi/ic_launcher.webp differ diff --git a/app/src/main/res/mipmap-xhdpi/ic_launcher_foreground.webp b/app/src/main/res/mipmap-xhdpi/ic_launcher_foreground.webp new file mode 100644 index 0000000..907bac8 Binary files /dev/null and b/app/src/main/res/mipmap-xhdpi/ic_launcher_foreground.webp differ diff --git a/app/src/main/res/mipmap-xhdpi/ic_launcher_round.webp b/app/src/main/res/mipmap-xhdpi/ic_launcher_round.webp index 1b9a695..3064e22 100644 Binary files a/app/src/main/res/mipmap-xhdpi/ic_launcher_round.webp and b/app/src/main/res/mipmap-xhdpi/ic_launcher_round.webp differ diff --git a/app/src/main/res/mipmap-xxhdpi/ic_launcher.webp b/app/src/main/res/mipmap-xxhdpi/ic_launcher.webp index 28d4b77..6e15368 100644 Binary files a/app/src/main/res/mipmap-xxhdpi/ic_launcher.webp and b/app/src/main/res/mipmap-xxhdpi/ic_launcher.webp differ diff --git a/app/src/main/res/mipmap-xxhdpi/ic_launcher_foreground.webp b/app/src/main/res/mipmap-xxhdpi/ic_launcher_foreground.webp new file mode 100644 index 0000000..77e60dd Binary files /dev/null and b/app/src/main/res/mipmap-xxhdpi/ic_launcher_foreground.webp differ diff --git a/app/src/main/res/mipmap-xxhdpi/ic_launcher_round.webp b/app/src/main/res/mipmap-xxhdpi/ic_launcher_round.webp index 9287f50..c95e65f 100644 Binary files a/app/src/main/res/mipmap-xxhdpi/ic_launcher_round.webp and b/app/src/main/res/mipmap-xxhdpi/ic_launcher_round.webp differ diff --git a/app/src/main/res/mipmap-xxxhdpi/ic_launcher.webp b/app/src/main/res/mipmap-xxxhdpi/ic_launcher.webp index aa7d642..9198ac2 100644 Binary files a/app/src/main/res/mipmap-xxxhdpi/ic_launcher.webp and b/app/src/main/res/mipmap-xxxhdpi/ic_launcher.webp differ diff --git a/app/src/main/res/mipmap-xxxhdpi/ic_launcher_foreground.webp b/app/src/main/res/mipmap-xxxhdpi/ic_launcher_foreground.webp new file mode 100644 index 0000000..8a09412 Binary files /dev/null and b/app/src/main/res/mipmap-xxxhdpi/ic_launcher_foreground.webp differ diff --git a/app/src/main/res/mipmap-xxxhdpi/ic_launcher_round.webp b/app/src/main/res/mipmap-xxxhdpi/ic_launcher_round.webp index 9126ae3..49f628e 100644 Binary files a/app/src/main/res/mipmap-xxxhdpi/ic_launcher_round.webp and b/app/src/main/res/mipmap-xxxhdpi/ic_launcher_round.webp differ diff --git a/app/src/main/res/values/arrays.xml b/app/src/main/res/values/arrays.xml new file mode 100644 index 0000000..f4e888d --- /dev/null +++ b/app/src/main/res/values/arrays.xml @@ -0,0 +1,13 @@ + + + + @string/pref_entries_pose_detector_performance_mode_fast + @string/pref_entries_pose_detector_performance_mode_accurate + + + + @string/pref_entry_values_pose_detector_performance_mode_fast + @string/pref_entry_values_pose_detector_performance_mode_accurate + + + \ No newline at end of file diff --git a/app/src/main/res/values/refs.xml b/app/src/main/res/values/refs.xml new file mode 100644 index 0000000..a6b3dae --- /dev/null +++ b/app/src/main/res/values/refs.xml @@ -0,0 +1,2 @@ + + \ No newline at end of file diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 8b00141..465add0 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -246,4 +246,57 @@ Please Enroll on a new program + + ps + Live preview settings + Still image settings + CameraX live preview settings + CameraXSource live preview settings + Detection Info + Pose Detection + + + + pckc + Camera + rcpvs + rcpts + fcpvs + fcpts + crctas + cfctas + clv + Rear camera preview size + Front camera preview size + CameraX rear camera target resolution + CameraX front camera target resolution + Enable live viewport + Do not block camera preview drawing on detection + + + Performance mode + lppdpm + sipdpm + Fast + Accurate + 1 + 2 + + + Prefer using GPU + pdpg + If enabled, GPU will be used as long as it is available, stable and returns correct results. In this case, the detector will not check if GPU is really faster than CPU. + + + Show in-frame likelihood + lppdsifl + sipdsifl + + + Visualize z value + pdvz + Rescale z value for visualization + pdrz + + \ No newline at end of file diff --git a/app/src/main/res/xml/pref_camera_view.xml b/app/src/main/res/xml/pref_camera_view.xml new file mode 100644 index 0000000..a2bcf7e --- /dev/null +++ b/app/src/main/res/xml/pref_camera_view.xml @@ -0,0 +1,79 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +