From ab67ca81f64a7c1da9f6f8baec7d71b679eb9f43 Mon Sep 17 00:00:00 2001 From: haohao <358297604@qq.com> Date: Fri, 13 Mar 2020 12:29:12 +0800 Subject: [PATCH 1/5] Implement an FfmpegVideoRenderer --- .gitignore | 2 + core_settings.gradle | 2 + extensions/ffmpegvideo/README.md | 86 ++++ extensions/ffmpegvideo/build.gradle | 90 ++++ extensions/ffmpegvideo/proguard-rules.txt | 7 + .../ffmpegvideo/src/main/AndroidManifest.xml | 17 + .../ext/ffmpeg/FFmpegRenderersFactory.java | 137 ++++++ .../ext/ffmpeg/FfmpegAudioDecoder.java | 230 +++++++++ .../ffmpeg/FfmpegAudioDecoderException.java | 32 ++ .../ext/ffmpeg/FfmpegAudioRenderer.java | 157 +++++++ .../exoplayer2/ext/ffmpeg/FfmpegLibrary.java | 164 +++++++ .../ext/ffmpeg/FfmpegVideoDecoder.java | 258 +++++++++++ .../ffmpeg/FfmpegVideoDecoderException.java | 30 ++ .../ext/ffmpeg/FfmpegVideoRenderer.java | 190 ++++++++ .../exoplayer2/ext/ffmpeg/package-info.java | 19 + .../ffmpegvideo/src/main/jni/CMakeLists.txt | 65 +++ .../src/main/jni/ffmpeg_audio_jni.cc | 360 +++++++++++++++ .../src/main/jni/ffmpeg_video_jni.cc | 436 ++++++++++++++++++ 18 files changed, 2282 insertions(+) create mode 100644 extensions/ffmpegvideo/README.md create mode 100644 extensions/ffmpegvideo/build.gradle create mode 100644 extensions/ffmpegvideo/proguard-rules.txt create mode 100644 extensions/ffmpegvideo/src/main/AndroidManifest.xml create mode 100644 extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FFmpegRenderersFactory.java create mode 100644 extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegAudioDecoder.java create mode 100644 extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegAudioDecoderException.java create mode 100644 extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegAudioRenderer.java create mode 100644 extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegLibrary.java create mode 100644 extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegVideoDecoder.java create mode 100644 extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegVideoDecoderException.java create mode 100644 extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegVideoRenderer.java create mode 100644 extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/package-info.java create mode 100644 extensions/ffmpegvideo/src/main/jni/CMakeLists.txt create mode 100644 extensions/ffmpegvideo/src/main/jni/ffmpeg_audio_jni.cc create mode 100644 extensions/ffmpegvideo/src/main/jni/ffmpeg_video_jni.cc diff --git a/.gitignore b/.gitignore index cb4cfaada1c..4c85a0f69f7 100644 --- a/.gitignore +++ b/.gitignore @@ -69,6 +69,8 @@ extensions/flac/src/main/jni/flac # FFmpeg extension extensions/ffmpeg/src/main/jni/ffmpeg +extensions/ffmpegvideo/.cxx +extensions/ffmpegvideo/src/main/jni/include # Cronet extension extensions/cronet/jniLibs/* diff --git a/core_settings.gradle b/core_settings.gradle index ac569331556..3f7e9956726 100644 --- a/core_settings.gradle +++ b/core_settings.gradle @@ -29,6 +29,7 @@ include modulePrefix + 'testutils' include modulePrefix + 'testdata' include modulePrefix + 'extension-av1' include modulePrefix + 'extension-ffmpeg' +include modulePrefix + 'extension-ffmpegvideo' include modulePrefix + 'extension-flac' include modulePrefix + 'extension-gvr' include modulePrefix + 'extension-ima' @@ -55,6 +56,7 @@ project(modulePrefix + 'testutils').projectDir = new File(rootDir, 'testutils') project(modulePrefix + 'testdata').projectDir = new File(rootDir, 'testdata') project(modulePrefix + 'extension-av1').projectDir = new File(rootDir, 'extensions/av1') project(modulePrefix + 'extension-ffmpeg').projectDir = new File(rootDir, 'extensions/ffmpeg') +project(modulePrefix + 'extension-ffmpegvideo').projectDir = new File(rootDir, 'extensions/ffmpegvideo') project(modulePrefix + 'extension-flac').projectDir = new File(rootDir, 'extensions/flac') project(modulePrefix + 'extension-gvr').projectDir = new File(rootDir, 'extensions/gvr') project(modulePrefix + 'extension-ima').projectDir = new File(rootDir, 'extensions/ima') diff --git a/extensions/ffmpegvideo/README.md b/extensions/ffmpegvideo/README.md new file mode 100644 index 00000000000..7cdab350ae9 --- /dev/null +++ b/extensions/ffmpegvideo/README.md @@ -0,0 +1,86 @@ +# ExoPlayer FFmpeg extension # + +The Ffmpeg extension provides `FfmpegAudioRenderer` and `FfmpegVideoRenderer`, which uses FFmpeg +native library to decode videos. + +***This extension is currently in its very infancy and is under development.*** + +***Whats working?*** +video supported codec: only H.264 +audio supported codec: same as original extension +supported surface type: video_decoder_gl_surface_view + +***On Plan:*** +- [ ] Support other surface types +- [ ] Organize the code +- [ ] Fix possible issues +- [ ] Video Decoder support Format.rotationDegrees +- [ ] Support other codecs + + +## License note ## + +Please note that whilst the code in this repository is licensed under +[Apache 2.0][], using this extension also requires building and including one or +more external libraries as described below. These are licensed separately. + +[Apache 2.0]: https://github.com/google/ExoPlayer/blob/release-v2/LICENSE + +## Build instructions (Linux, macOS) ## + +To use this extension you need to clone the ExoPlayer repository and depend on +its modules locally. Instructions for doing this can be found in ExoPlayer's +[top level README][]. + +I provided the compiled FFmpeg [*.so files][] and [header files][]. You need to copy +the .so files to the `src/main/libs` directory and the header files to +the `src/main/jni/include` directory. Of course you can also compile it yourself. + + +## Using the extension ## + +Like av1 extension, pass `EXTENSION_RENDERER_MODE_PREFER`, use `FFmpegRenderersFactory` +instead of `DefaultRenderersFactory` to create `FfmpegVideoRenderer` and `FfmpegAudioRenderer`. +Then you can observe the related logs of `EventLogger#decoderInitialized` in logcat +to determine whether the ffmpeg extension is used correctly. + +## Using the extension in the demo application ## + +To try out playback using the extension in the [demo application][], see +[enabling extension decoders][]. + +use `FFmpegRenderersFactory` instead of `DefaultRenderersFactory`. + +[demo application]: https://exoplayer.dev/demo-application.html +[enabling extension decoders]: https://exoplayer.dev/demo-application.html#enabling-extension-decoders + +## Rendering options ## + +There are two possibilities for rendering the output `Libgav1VideoRenderer` +gets from the libgav1 decoder: + +* GL rendering using GL shader for color space conversion + * If you are using `SimpleExoPlayer` with `PlayerView`, enable this option by + setting `surface_type` of `PlayerView` to be + `video_decoder_gl_surface_view`. + * Otherwise, enable this option by sending `Libgav1VideoRenderer` a message + of type `C.MSG_SET_VIDEO_DECODER_OUTPUT_BUFFER_RENDERER` with an instance of + `VideoDecoderOutputBufferRenderer` as its object. + +* Native rendering using `ANativeWindow` + * If you are using `SimpleExoPlayer` with `PlayerView`, this option is enabled + by default. + * Otherwise, enable this option by sending `Libgav1VideoRenderer` a message of + type `C.MSG_SET_SURFACE` with an instance of `SurfaceView` as its object. + +Note: Although the default option uses `ANativeWindow`, based on our testing the +GL rendering mode has better performance, so should be preferred + +## Links ## + +* [Javadoc][]: Classes matching `com.google.android.exoplayer2.ext.av1.*` + belong to this module. + +[Javadoc]: https://exoplayer.dev/doc/reference/index.html +[*.so files]: https://drive.google.com/open?id=14v4tz5L_jU7di3xWrY-uhuS7K5mcwj3g +[header files]: https://drive.google.com/open?id=1dDZ9R4cLPpgcHOCoUpClrOlqnGL2UTSr diff --git a/extensions/ffmpegvideo/build.gradle b/extensions/ffmpegvideo/build.gradle new file mode 100644 index 00000000000..40278a439ef --- /dev/null +++ b/extensions/ffmpegvideo/build.gradle @@ -0,0 +1,90 @@ +// Copyright (C) 2019 The Android Open Source Project +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +apply from: '../../constants.gradle' +apply plugin: 'com.android.library' + +android { + compileSdkVersion project.ext.compileSdkVersion + + compileOptions { + sourceCompatibility JavaVersion.VERSION_1_8 + targetCompatibility JavaVersion.VERSION_1_8 + } + + defaultConfig { + minSdkVersion project.ext.minSdkVersion + targetSdkVersion project.ext.targetSdkVersion + consumerProguardFiles 'proguard-rules.txt' + + externalNativeBuild { + cmake { + // Debug CMake build type causes video frames to drop, + // so native library should always use Release build type. + arguments "-DCMAKE_BUILD_TYPE=Release" + targets "ffmpegJNI" + } + } + } + + externalNativeBuild { + cmake { + version '3.10.2' + path "src/main/jni/CMakeLists.txt" + } + } + + buildTypes { + debug { + ndk { + abiFilters 'arm64-v8a'/*, 'x86_64'*/ + } + } + } + + // This option resolves the problem of finding libgav1JNI.so + // on multiple paths. The first one found is picked. + packagingOptions { + pickFirst 'lib/arm64-v8a/libffmpegJNI.so' + pickFirst 'lib/armeabi-v7a/libffmpegJNI.so' + pickFirst 'lib/x86/libffmpegJNI.so' + pickFirst 'lib/x86_64/libffmpegJNI.so' + } + + sourceSets.main { + // As native JNI library build is invoked from gradle, this is + // not needed. However, it exposes the built library and keeps + // consistency with the other extensions. + jniLibs.srcDir 'src/main/libs' + } +} + +// Configure the native build only if libgav1 is present, to avoid gradle sync +// failures if libgav1 hasn't been checked out according to the README and CMake +// isn't installed. +//if (project.file('src/main/jni/libffmpeg').exists()) { +// android.externalNativeBuild.cmake.path = 'src/main/jni/CMakeLists.txt' +// android.externalNativeBuild.cmake.version = '3.7.1+' +//} + +dependencies { + implementation project(modulePrefix + 'library-core') + implementation 'androidx.annotation:annotation:' + androidxAnnotationVersion + compileOnly 'org.checkerframework:checker-qual:' + checkerframeworkVersion + compileOnly 'org.jetbrains.kotlin:kotlin-annotations-jvm:' + kotlinAnnotationsVersion +} + +ext { + javadocTitle = 'FFmpeg video extension' +} +apply from: '../../javadoc_library.gradle' diff --git a/extensions/ffmpegvideo/proguard-rules.txt b/extensions/ffmpegvideo/proguard-rules.txt new file mode 100644 index 00000000000..9d73f7e2b58 --- /dev/null +++ b/extensions/ffmpegvideo/proguard-rules.txt @@ -0,0 +1,7 @@ +# Proguard rules specific to the AV1 extension. + +# This prevents the names of native methods from being obfuscated. +-keepclasseswithmembernames class * { + native ; +} + diff --git a/extensions/ffmpegvideo/src/main/AndroidManifest.xml b/extensions/ffmpegvideo/src/main/AndroidManifest.xml new file mode 100644 index 00000000000..d53bca4ca22 --- /dev/null +++ b/extensions/ffmpegvideo/src/main/AndroidManifest.xml @@ -0,0 +1,17 @@ + + + + diff --git a/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FFmpegRenderersFactory.java b/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FFmpegRenderersFactory.java new file mode 100644 index 00000000000..229589dff23 --- /dev/null +++ b/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FFmpegRenderersFactory.java @@ -0,0 +1,137 @@ +package com.google.android.exoplayer2.ext.ffmpeg; + +import android.content.Context; +import android.os.Handler; +import com.google.android.exoplayer2.DefaultRenderersFactory; +import com.google.android.exoplayer2.Renderer; +import com.google.android.exoplayer2.audio.AudioCapabilities; +import com.google.android.exoplayer2.audio.AudioProcessor; +import com.google.android.exoplayer2.audio.AudioRendererEventListener; +import com.google.android.exoplayer2.audio.DefaultAudioSink; +import com.google.android.exoplayer2.audio.MediaCodecAudioRenderer; +import com.google.android.exoplayer2.mediacodec.MediaCodecSelector; +import com.google.android.exoplayer2.util.Log; +import com.google.android.exoplayer2.video.MediaCodecVideoRenderer; +import com.google.android.exoplayer2.video.VideoRendererEventListener; +import java.lang.reflect.Constructor; +import java.util.ArrayList; + +public class FFmpegRenderersFactory extends DefaultRenderersFactory { + + private static final String TAG = "FFmpegRenderersFactory"; + + public FFmpegRenderersFactory(Context context) { + super(context); + } + + @Override + protected void buildVideoRenderers( + Context context, + @ExtensionRendererMode int extensionRendererMode, + MediaCodecSelector mediaCodecSelector, + boolean enableDecoderFallback, + Handler eventHandler, + VideoRendererEventListener eventListener, + long allowedVideoJoiningTimeMs, + ArrayList out) { + MediaCodecVideoRenderer videoRenderer = + new MediaCodecVideoRenderer( + context, + mediaCodecSelector, + allowedVideoJoiningTimeMs, + enableDecoderFallback, + eventHandler, + eventListener, + MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY); + out.add(videoRenderer); + + if (extensionRendererMode == EXTENSION_RENDERER_MODE_OFF) { + return; + } + int extensionRendererIndex = out.size(); + if (extensionRendererMode == EXTENSION_RENDERER_MODE_PREFER) { + extensionRendererIndex--; + } + + try { + // Full class names used for constructor args so the LINT rule triggers if any of them move. + // LINT.IfChange + Class clazz = Class + .forName("com.google.android.exoplayer2.ext.ffmpeg.FfmpegVideoRenderer"); + Constructor constructor = + clazz.getConstructor( + long.class, + Handler.class, + com.google.android.exoplayer2.video.VideoRendererEventListener.class, + int.class); + // LINT.ThenChange(../../../../../../../proguard-rules.txt) + Renderer renderer = + (Renderer) + constructor.newInstance( + allowedVideoJoiningTimeMs, + eventHandler, + eventListener, + MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY); + out.add(extensionRendererIndex++, renderer); + Log.i(TAG, "Loaded FfmpegVideoRenderer."); + } catch (ClassNotFoundException e) { + // Expected if the app was built without the extension. + } catch (Exception e) { + // The extension is present, but instantiation failed. + throw new RuntimeException("Error instantiating Ffmpeg extension", e); + } + + } + + @Override + protected void buildAudioRenderers( + Context context, + int extensionRendererMode, + MediaCodecSelector mediaCodecSelector, + boolean enableDecoderFallback, + AudioProcessor[] audioProcessors, + Handler eventHandler, + AudioRendererEventListener eventListener, + ArrayList out) { + MediaCodecAudioRenderer audioRenderer = + new MediaCodecAudioRenderer( + context, + mediaCodecSelector, + enableDecoderFallback, + eventHandler, + eventListener, + new DefaultAudioSink(AudioCapabilities.getCapabilities(context), audioProcessors)); + out.add(audioRenderer); + + if (extensionRendererMode == EXTENSION_RENDERER_MODE_OFF) { + return; + } + int extensionRendererIndex = out.size(); + if (extensionRendererMode == EXTENSION_RENDERER_MODE_PREFER) { + extensionRendererIndex--; + } + + try { + // Full class names used for constructor args so the LINT rule triggers if any of them move. + // LINT.IfChange + Class clazz = + Class.forName("com.google.android.exoplayer2.ext.ffmpeg.FfmpegAudioRenderer"); + Constructor constructor = + clazz.getConstructor( + android.os.Handler.class, + com.google.android.exoplayer2.audio.AudioRendererEventListener.class, + com.google.android.exoplayer2.audio.AudioProcessor[].class); + // LINT.ThenChange(../../../../../../../proguard-rules.txt) + Renderer renderer = + (Renderer) constructor.newInstance(eventHandler, eventListener, audioProcessors); + out.add(extensionRendererIndex++, renderer); + Log.i(TAG, "Loaded FfmpegAudioRenderer."); + } catch (ClassNotFoundException e) { + // Expected if the app was built without the extension. + } catch (Exception e) { + // The extension is present, but instantiation failed. + throw new RuntimeException("Error instantiating FFmpeg extension", e); + } + } + +} diff --git a/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegAudioDecoder.java b/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegAudioDecoder.java new file mode 100644 index 00000000000..f0ac07b8aa7 --- /dev/null +++ b/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegAudioDecoder.java @@ -0,0 +1,230 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.ext.ffmpeg; + +import androidx.annotation.Nullable; +import com.google.android.exoplayer2.C; +import com.google.android.exoplayer2.Format; +import com.google.android.exoplayer2.decoder.DecoderInputBuffer; +import com.google.android.exoplayer2.decoder.SimpleDecoder; +import com.google.android.exoplayer2.decoder.SimpleOutputBuffer; +import com.google.android.exoplayer2.util.Assertions; +import com.google.android.exoplayer2.util.MimeTypes; +import com.google.android.exoplayer2.util.ParsableByteArray; +import com.google.android.exoplayer2.util.Util; +import java.nio.ByteBuffer; +import java.util.List; + +/** + * FFmpeg audio decoder. + */ +/* package */ final class FfmpegAudioDecoder extends + SimpleDecoder { + + // Output buffer sizes when decoding PCM mu-law streams, which is the maximum FFmpeg outputs. + private static final int OUTPUT_BUFFER_SIZE_16BIT = 65536; + private static final int OUTPUT_BUFFER_SIZE_32BIT = OUTPUT_BUFFER_SIZE_16BIT * 2; + + // Error codes matching ffmpeg_audio_jni.cc. + private static final int DECODER_ERROR_INVALID_DATA = -1; + private static final int DECODER_ERROR_OTHER = -2; + + private final String codecName; + @Nullable private final byte[] extraData; + private final @C.Encoding int encoding; + private final int outputBufferSize; + + private long nativeContext; // May be reassigned on resetting the codec. + private boolean hasOutputFormat; + private volatile int channelCount; + private volatile int sampleRate; + + public FfmpegAudioDecoder( + int numInputBuffers, + int numOutputBuffers, + int initialInputBufferSize, + Format format, + boolean outputFloat) + throws FfmpegAudioDecoderException { + super(new DecoderInputBuffer[numInputBuffers], new SimpleOutputBuffer[numOutputBuffers]); + if (!FfmpegLibrary.isAvailable()) { + throw new FfmpegAudioDecoderException("Failed to load decoder native libraries."); + } + Assertions.checkNotNull(format.sampleMimeType); + codecName = + Assertions.checkNotNull(FfmpegLibrary.getAudioCodecName(format.sampleMimeType)); + extraData = getExtraData(format.sampleMimeType, format.initializationData); + encoding = outputFloat ? C.ENCODING_PCM_FLOAT : C.ENCODING_PCM_16BIT; + outputBufferSize = outputFloat ? OUTPUT_BUFFER_SIZE_32BIT : OUTPUT_BUFFER_SIZE_16BIT; + nativeContext = + ffmpegInitialize(codecName, extraData, outputFloat, format.sampleRate, format.channelCount); + if (nativeContext == 0) { + throw new FfmpegAudioDecoderException("Initialization failed."); + } + setInitialInputBufferSize(initialInputBufferSize); + } + + @Override + public String getName() { + return "ffmpeg" + FfmpegLibrary.getVersion() + "-" + codecName; + } + + @Override + protected DecoderInputBuffer createInputBuffer() { + return new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DIRECT); + } + + @Override + protected SimpleOutputBuffer createOutputBuffer() { + return new SimpleOutputBuffer(this); + } + + @Override + protected FfmpegAudioDecoderException createUnexpectedDecodeException(Throwable error) { + return new FfmpegAudioDecoderException("Unexpected decode error", error); + } + + @Override + protected @Nullable FfmpegAudioDecoderException decode( + DecoderInputBuffer inputBuffer, SimpleOutputBuffer outputBuffer, boolean reset) { + if (reset) { + nativeContext = ffmpegReset(nativeContext, extraData); + if (nativeContext == 0) { + return new FfmpegAudioDecoderException("Error resetting (see logcat)."); + } + } + ByteBuffer inputData = Util.castNonNull(inputBuffer.data); + int inputSize = inputData.limit(); + ByteBuffer outputData = outputBuffer.init(inputBuffer.timeUs, outputBufferSize); + int result = ffmpegDecode(nativeContext, inputData, inputSize, outputData, outputBufferSize); + if (result == DECODER_ERROR_INVALID_DATA) { + // Treat invalid data errors as non-fatal to match the behavior of MediaCodec. No output will + // be produced for this buffer, so mark it as decode-only to ensure that the audio sink's + // position is reset when more audio is produced. + outputBuffer.setFlags(C.BUFFER_FLAG_DECODE_ONLY); + return null; + } else if (result == DECODER_ERROR_OTHER) { + return new FfmpegAudioDecoderException("Error decoding (see logcat)."); + } + if (!hasOutputFormat) { + channelCount = ffmpegGetChannelCount(nativeContext); + sampleRate = ffmpegGetSampleRate(nativeContext); + if (sampleRate == 0 && "alac".equals(codecName)) { + Assertions.checkNotNull(extraData); + // ALAC decoder did not set the sample rate in earlier versions of FFMPEG. + // See https://trac.ffmpeg.org/ticket/6096 + ParsableByteArray parsableExtraData = new ParsableByteArray(extraData); + parsableExtraData.setPosition(extraData.length - 4); + sampleRate = parsableExtraData.readUnsignedIntToInt(); + } + hasOutputFormat = true; + } + outputData.position(0); + outputData.limit(result); + return null; + } + + @Override + public void release() { + super.release(); + ffmpegRelease(nativeContext); + nativeContext = 0; + } + + /** Returns the channel count of output audio. */ + public int getChannelCount() { + return channelCount; + } + + /** Returns the sample rate of output audio. */ + public int getSampleRate() { + return sampleRate; + } + + /** + * Returns the encoding of output audio. + */ + public @C.Encoding int getEncoding() { + return encoding; + } + + /** + * Returns FFmpeg-compatible codec-specific initialization data ("extra data"), or {@code null} if + * not required. + */ + private static @Nullable byte[] getExtraData(String mimeType, List initializationData) { + switch (mimeType) { + case MimeTypes.AUDIO_AAC: + case MimeTypes.AUDIO_OPUS: + return initializationData.get(0); + case MimeTypes.AUDIO_ALAC: + return getAlacExtraData(initializationData); + case MimeTypes.AUDIO_VORBIS: + return getVorbisExtraData(initializationData); + default: + // Other codecs do not require extra data. + return null; + } + } + + private static byte[] getAlacExtraData(List initializationData) { + // FFmpeg's ALAC decoder expects an ALAC atom, which contains the ALAC "magic cookie", as extra + // data. initializationData[0] contains only the magic cookie, and so we need to package it into + // an ALAC atom. See: + // https://ffmpeg.org/doxygen/0.6/alac_8c.html + // https://github.com/macosforge/alac/blob/master/ALACMagicCookieDescription.txt + byte[] magicCookie = initializationData.get(0); + int alacAtomLength = 12 + magicCookie.length; + ByteBuffer alacAtom = ByteBuffer.allocate(alacAtomLength); + alacAtom.putInt(alacAtomLength); + alacAtom.putInt(0x616c6163); // type=alac + alacAtom.putInt(0); // version=0, flags=0 + alacAtom.put(magicCookie, /* offset= */ 0, magicCookie.length); + return alacAtom.array(); + } + + private static byte[] getVorbisExtraData(List initializationData) { + byte[] header0 = initializationData.get(0); + byte[] header1 = initializationData.get(1); + byte[] extraData = new byte[header0.length + header1.length + 6]; + extraData[0] = (byte) (header0.length >> 8); + extraData[1] = (byte) (header0.length & 0xFF); + System.arraycopy(header0, 0, extraData, 2, header0.length); + extraData[header0.length + 2] = 0; + extraData[header0.length + 3] = 0; + extraData[header0.length + 4] = (byte) (header1.length >> 8); + extraData[header0.length + 5] = (byte) (header1.length & 0xFF); + System.arraycopy(header1, 0, extraData, header0.length + 6, header1.length); + return extraData; + } + + private native long ffmpegInitialize( + String codecName, + @Nullable byte[] extraData, + boolean outputFloat, + int rawSampleRate, + int rawChannelCount); + + private native int ffmpegDecode(long context, ByteBuffer inputData, int inputSize, + ByteBuffer outputData, int outputSize); + private native int ffmpegGetChannelCount(long context); + private native int ffmpegGetSampleRate(long context); + + private native long ffmpegReset(long context, @Nullable byte[] extraData); + + private native void ffmpegRelease(long context); + +} diff --git a/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegAudioDecoderException.java b/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegAudioDecoderException.java new file mode 100644 index 00000000000..82aac5a9ef4 --- /dev/null +++ b/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegAudioDecoderException.java @@ -0,0 +1,32 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.ext.ffmpeg; + +import com.google.android.exoplayer2.audio.AudioDecoderException; + +/** + * Thrown when an FFmpeg decoder error occurs. + */ +public final class FfmpegAudioDecoderException extends AudioDecoderException { + + /* package */ FfmpegAudioDecoderException(String message) { + super(message); + } + + /* package */ FfmpegAudioDecoderException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegAudioRenderer.java b/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegAudioRenderer.java new file mode 100644 index 00000000000..726a367160b --- /dev/null +++ b/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegAudioRenderer.java @@ -0,0 +1,157 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.ext.ffmpeg; + +import android.os.Handler; +import androidx.annotation.Nullable; +import com.google.android.exoplayer2.C; +import com.google.android.exoplayer2.Format; +import com.google.android.exoplayer2.audio.AudioProcessor; +import com.google.android.exoplayer2.audio.AudioRendererEventListener; +import com.google.android.exoplayer2.audio.AudioSink; +import com.google.android.exoplayer2.audio.DefaultAudioSink; +import com.google.android.exoplayer2.audio.SimpleDecoderAudioRenderer; +import com.google.android.exoplayer2.drm.ExoMediaCrypto; +import com.google.android.exoplayer2.util.Assertions; +import com.google.android.exoplayer2.util.MimeTypes; +import org.checkerframework.checker.nullness.qual.MonotonicNonNull; + +/** + * Decodes and renders audio using FFmpeg. + */ +public final class FfmpegAudioRenderer extends SimpleDecoderAudioRenderer { + + /** The number of input and output buffers. */ + private static final int NUM_BUFFERS = 16; + /** The default input buffer size. */ + private static final int DEFAULT_INPUT_BUFFER_SIZE = 960 * 6; + + private final boolean enableFloatOutput; + + private @MonotonicNonNull FfmpegAudioDecoder decoder; + + public FfmpegAudioRenderer() { + this(/* eventHandler= */ null, /* eventListener= */ null); + } + + /** + * @param eventHandler A handler to use when delivering events to {@code eventListener}. May be + * null if delivery of events is not required. + * @param eventListener A listener of events. May be null if delivery of events is not required. + * @param audioProcessors Optional {@link AudioProcessor}s that will process audio before output. + */ + public FfmpegAudioRenderer( + @Nullable Handler eventHandler, + @Nullable AudioRendererEventListener eventListener, + AudioProcessor... audioProcessors) { + this( + eventHandler, + eventListener, + new DefaultAudioSink(/* audioCapabilities= */ null, audioProcessors), + /* enableFloatOutput= */ false); + } + + /** + * @param eventHandler A handler to use when delivering events to {@code eventListener}. May be + * null if delivery of events is not required. + * @param eventListener A listener of events. May be null if delivery of events is not required. + * @param audioSink The sink to which audio will be output. + * @param enableFloatOutput Whether to enable 32-bit float audio format, if supported on the + * device/build and if the input format may have bit depth higher than 16-bit. When using + * 32-bit float output, any audio processing will be disabled, including playback speed/pitch + * adjustment. + */ + public FfmpegAudioRenderer( + @Nullable Handler eventHandler, + @Nullable AudioRendererEventListener eventListener, + AudioSink audioSink, + boolean enableFloatOutput) { + super( + eventHandler, + eventListener, + audioSink); + this.enableFloatOutput = enableFloatOutput; + } + + @Override + @FormatSupport + protected int supportsFormatInternal(Format format) { + Assertions.checkNotNull(format.sampleMimeType); + if (!FfmpegLibrary.isAvailable()) { + return FORMAT_UNSUPPORTED_TYPE; + } else if (!FfmpegLibrary.supportsAudioFormat(format.sampleMimeType) || !isOutputSupported(format)) { + return FORMAT_UNSUPPORTED_SUBTYPE; + } else if (format.drmInitData != null && format.exoMediaCryptoType == null) { + return FORMAT_UNSUPPORTED_DRM; + } else { + return FORMAT_HANDLED; + } + } + + @Override + @AdaptiveSupport + public final int supportsMixedMimeTypeAdaptation() { + return ADAPTIVE_NOT_SEAMLESS; + } + + @Override + protected FfmpegAudioDecoder createDecoder(Format format, @Nullable ExoMediaCrypto mediaCrypto) + throws FfmpegAudioDecoderException { + int initialInputBufferSize = + format.maxInputSize != Format.NO_VALUE ? format.maxInputSize : DEFAULT_INPUT_BUFFER_SIZE; + decoder = + new FfmpegAudioDecoder( + NUM_BUFFERS, NUM_BUFFERS, initialInputBufferSize, format, shouldUseFloatOutput(format)); + return decoder; + } + + @Override + public Format getOutputFormat() { + Assertions.checkNotNull(decoder); + return new Format.Builder() + .setSampleMimeType(MimeTypes.AUDIO_RAW) + .setChannelCount(decoder.getChannelCount()) + .setSampleRate(decoder.getSampleRate()) + .setPcmEncoding(decoder.getEncoding()) + .build(); + } + + private boolean isOutputSupported(Format inputFormat) { + return shouldUseFloatOutput(inputFormat) + || supportsOutput(inputFormat.channelCount, C.ENCODING_PCM_16BIT); + } + + private boolean shouldUseFloatOutput(Format inputFormat) { + Assertions.checkNotNull(inputFormat.sampleMimeType); + if (!enableFloatOutput || !supportsOutput(inputFormat.channelCount, C.ENCODING_PCM_FLOAT)) { + return false; + } + switch (inputFormat.sampleMimeType) { + case MimeTypes.AUDIO_RAW: + // For raw audio, output in 32-bit float encoding if the bit depth is > 16-bit. + return inputFormat.pcmEncoding == C.ENCODING_PCM_24BIT + || inputFormat.pcmEncoding == C.ENCODING_PCM_32BIT + || inputFormat.pcmEncoding == C.ENCODING_PCM_FLOAT; + case MimeTypes.AUDIO_AC3: + // AC-3 is always 16-bit, so there is no point outputting in 32-bit float encoding. + return false; + default: + // For all other formats, assume that it's worth using 32-bit float encoding. + return true; + } + } + +} diff --git a/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegLibrary.java b/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegLibrary.java new file mode 100644 index 00000000000..6bcbf3693f7 --- /dev/null +++ b/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegLibrary.java @@ -0,0 +1,164 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.ext.ffmpeg; + +import androidx.annotation.Nullable; +import com.google.android.exoplayer2.ExoPlayerLibraryInfo; +import com.google.android.exoplayer2.util.LibraryLoader; +import com.google.android.exoplayer2.util.Log; +import com.google.android.exoplayer2.util.MimeTypes; + +/** + * Configures and queries the underlying native library. + */ +public final class FfmpegLibrary { + + static { + ExoPlayerLibraryInfo.registerModule("goog.exo.ffmpegvideo"); + } + + private static final String TAG = "FfmpegLibrary"; + + private static final LibraryLoader LOADER = + new LibraryLoader("ffmpeg", "ffmpegJNI"); + + private FfmpegLibrary() {} + + /** + * Override the names of the FFmpeg native libraries. If an application wishes to call this + * method, it must do so before calling any other method defined by this class, and before + * instantiating a {@link FfmpegAudioRenderer} instance. + * + * @param libraries The names of the FFmpeg native libraries. + */ + public static void setLibraries(String... libraries) { + LOADER.setLibraries(libraries); + } + + /** + * Returns whether the underlying library is available, loading it if necessary. + */ + public static boolean isAvailable() { + return LOADER.isAvailable(); + } + + /** Returns the version of the underlying library if available, or null otherwise. */ + public static @Nullable String getVersion() { + return isAvailable() ? ffmpegGetVersion() : null; + } + + /** + * Returns whether the underlying library supports the specified MIME type. + * + * @param mimeType The MIME type to check. + */ + public static boolean supportsAudioFormat(String mimeType) { + if (!isAvailable()) { + return false; + } + String codecName = getAudioCodecName(mimeType); + if (codecName == null) { + return false; + } + if (!ffmpegHasDecoder(codecName)) { + Log.w(TAG, "No " + codecName + " decoder available. Check the FFmpeg build configuration."); + return false; + } + return true; + } + + /** + * Returns whether the underlying library supports the specified MIME type. + * + * @param mimeType The MIME type to check. + */ + public static boolean supportsVideoFormat(String mimeType) { + if (!isAvailable()) { + return false; + } + String codecName = getVideoCodecName(mimeType); + if (codecName == null) { + return false; + } + if (!ffmpegHasDecoder(codecName)) { + Log.w(TAG, "No " + codecName + " decoder available. Check the FFmpeg build configuration."); + return false; + } + return true; + } + + /** + * Returns the name of the FFmpeg decoder that could be used to decode the format, or {@code null} + * if it's unsupported. + */ + /* package */ static @Nullable String getAudioCodecName(String mimeType) { + switch (mimeType) { + case MimeTypes.AUDIO_AAC: + return "aac"; + case MimeTypes.AUDIO_MPEG: + case MimeTypes.AUDIO_MPEG_L1: + case MimeTypes.AUDIO_MPEG_L2: + return "mp3"; + case MimeTypes.AUDIO_AC3: + return "ac3"; + case MimeTypes.AUDIO_E_AC3: + case MimeTypes.AUDIO_E_AC3_JOC: + return "eac3"; + case MimeTypes.AUDIO_TRUEHD: + return "truehd"; + case MimeTypes.AUDIO_DTS: + case MimeTypes.AUDIO_DTS_HD: + return "dca"; + case MimeTypes.AUDIO_VORBIS: + return "vorbis"; + case MimeTypes.AUDIO_OPUS: + return "opus"; + case MimeTypes.AUDIO_AMR_NB: + return "amrnb"; + case MimeTypes.AUDIO_AMR_WB: + return "amrwb"; + case MimeTypes.AUDIO_FLAC: + return "flac"; + case MimeTypes.AUDIO_ALAC: + return "alac"; + case MimeTypes.AUDIO_MLAW: + return "pcm_mulaw"; + case MimeTypes.AUDIO_ALAW: + return "pcm_alaw"; + default: + return null; + } + } + + /** + * Returns the name of the FFmpeg decoder that could be used to decode the format, or {@code null} + * if it's unsupported. + */ + /* package */ static @Nullable String getVideoCodecName(String mimeType) { + switch (mimeType) { + case MimeTypes.VIDEO_H264: + return "h264"; + case MimeTypes.VIDEO_H265: + return "hevc"; + default: + return null; + } + } + + private static native String ffmpegGetVersion(); + private static native boolean ffmpegHasDecoder(String codecName); + +} diff --git a/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegVideoDecoder.java b/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegVideoDecoder.java new file mode 100644 index 00000000000..874299453f3 --- /dev/null +++ b/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegVideoDecoder.java @@ -0,0 +1,258 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.ext.ffmpeg; + +import android.util.Log; +import android.view.Surface; +import androidx.annotation.Nullable; +import com.google.android.exoplayer2.C; +import com.google.android.exoplayer2.Format; +import com.google.android.exoplayer2.decoder.SimpleDecoder; +import com.google.android.exoplayer2.util.Assertions; +import com.google.android.exoplayer2.util.MimeTypes; +import com.google.android.exoplayer2.util.Util; +import com.google.android.exoplayer2.video.VideoDecoderInputBuffer; +import com.google.android.exoplayer2.video.VideoDecoderOutputBuffer; +import java.nio.ByteBuffer; +import java.util.List; + +/** + * Ffmpeg Video decoder. + */ +/* package */ final class FfmpegVideoDecoder + extends + SimpleDecoder { + + // Error codes matching ffmpeg_video_jni.cc. + private static final int DECODER_ERROR_INVALID_DATA = -1; + private static final int DECODER_ERROR_OTHER = -2; + private static final int DECODER_ERROR_READ_FRAME = -3; + private static final int DECODER_ERROR_SEND_PACKET = -4; + + private final String codecName; + private long nativeContext; + @Nullable private final byte[] extraData; + private Format format; + + @C.VideoOutputMode private volatile int outputMode; + + /** + * Creates a Ffmpeg video Decoder. + * + * @param numInputBuffers Number of input buffers. + * @param numOutputBuffers Number of output buffers. + * @param initialInputBufferSize The initial size of each input buffer, in bytes. + * @param threads Number of threads libgav1 will use to decode. + * @throws FfmpegVideoDecoderException Thrown if an exception occurs when initializing the + * decoder. + */ + public FfmpegVideoDecoder( + int numInputBuffers, int numOutputBuffers, int initialInputBufferSize, int threads, Format format) + throws FfmpegVideoDecoderException { + super( + new VideoDecoderInputBuffer[numInputBuffers], + new VideoDecoderOutputBuffer[numOutputBuffers]); + if (!FfmpegLibrary.isAvailable()) { + throw new FfmpegVideoDecoderException("Failed to load decoder native library."); + } + codecName = Assertions.checkNotNull(FfmpegLibrary.getVideoCodecName(format.sampleMimeType)); + extraData = getExtraData(format.sampleMimeType, format.initializationData); + this.format = format; + nativeContext = ffmpegInitialize(codecName, extraData, threads); + if (nativeContext == 0) { + throw new FfmpegVideoDecoderException("Failed to initialize decoder."); + } + setInitialInputBufferSize(initialInputBufferSize); + } + + /** + * Returns FFmpeg-compatible codec-specific initialization data ("extra data"), or {@code null} if + * not required. + */ + @Nullable + private static byte[] getExtraData(String mimeType, List initializationData) { + switch (mimeType) { + case MimeTypes.VIDEO_H264: + byte[] sps = initializationData.get(0); + byte[] pps = initializationData.get(1); + byte[] extraData = new byte[sps.length + pps.length]; + System.arraycopy(sps, 0, extraData, 0, sps.length); + System.arraycopy(pps, 0, extraData, sps.length, pps.length); + return extraData; + case MimeTypes.VIDEO_H265: + return initializationData.get(0); + default: + // Other codecs do not require extra data. + return null; + } + } + + @Override + public String getName() { + return "ffmpeg" + FfmpegLibrary.getVersion() + "-" + codecName; + } + + /** + * Sets the output mode for frames rendered by the decoder. + * + * @param outputMode The output mode. + */ + public void setOutputMode(@C.VideoOutputMode int outputMode) { + this.outputMode = outputMode; + } + + @Override + protected VideoDecoderInputBuffer createInputBuffer() { + return new VideoDecoderInputBuffer(); + } + + @Override + protected VideoDecoderOutputBuffer createOutputBuffer() { + return new VideoDecoderOutputBuffer(this::releaseOutputBuffer); + } + + @Override + @Nullable + protected FfmpegVideoDecoderException decode( + VideoDecoderInputBuffer inputBuffer, VideoDecoderOutputBuffer outputBuffer, boolean reset) { + if (reset) { + nativeContext = ffmpegReset(nativeContext); + if (nativeContext == 0) { + return new FfmpegVideoDecoderException("Error resetting (see logcat)."); + } + } + + // send packet + ByteBuffer inputData = Util.castNonNull(inputBuffer.data); + int inputSize = inputData.limit(); + // enqueue origin data + boolean needSendAgain = false; + int sendPacketResult = ffmpegSendPacket(nativeContext, inputData, inputSize, + inputBuffer.timeUs); + if (sendPacketResult == DECODER_ERROR_INVALID_DATA) { + outputBuffer.setFlags(C.BUFFER_FLAG_DECODE_ONLY); + return null; + } else if (sendPacketResult == DECODER_ERROR_READ_FRAME) { + // need read frame + needSendAgain = true; + } else if (sendPacketResult == DECODER_ERROR_OTHER) { + return new FfmpegVideoDecoderException("ffmpegDecode error: (see logcat)"); + } + + // receive frame + boolean decodeOnly = inputBuffer.isDecodeOnly(); + // We need to dequeue the decoded frame from the decoder even when the input data is + // decode-only. + int getFrameResult = ffmpegReceiveFrame(nativeContext, outputMode, outputBuffer, decodeOnly); + if (getFrameResult == DECODER_ERROR_SEND_PACKET) { + return null; + } else if (getFrameResult == DECODER_ERROR_OTHER) { + return new FfmpegVideoDecoderException("ffmpegDecode error: (see logcat)"); + } + + if (getFrameResult == DECODER_ERROR_INVALID_DATA) { + outputBuffer.addFlag(C.BUFFER_FLAG_DECODE_ONLY); + } + + if (!decodeOnly) { + outputBuffer.colorInfo = inputBuffer.colorInfo; + } + + if (needSendAgain) { + Log.e("ffmpeg_jni", "timeUs=" + inputBuffer.timeUs + ", " + "nendSendAagin"); + } + + return null; + } + + @Override + protected FfmpegVideoDecoderException createUnexpectedDecodeException(Throwable error) { + return new FfmpegVideoDecoderException("Unexpected decode error", error); + } + + @Override + public void release() { + super.release(); + ffmpegRelease(nativeContext); + nativeContext = 0; + } + + @Override + protected void releaseOutputBuffer(VideoDecoderOutputBuffer buffer) { + // Decode only frames do not acquire a reference on the internal decoder buffer and thus do not + // require a call to vpxReleaseFrame. +// if (outputMode == C.VIDEO_OUTPUT_MODE_SURFACE_YUV && !buffer.isDecodeOnly()) { +// gav1ReleaseFrame(nativeContext, buffer); +// } + super.releaseOutputBuffer(buffer); + } + + /** + * Renders output buffer to the given surface. Must only be called when in {@link + * C#VIDEO_OUTPUT_MODE_SURFACE_YUV} mode. + * + * @param outputBuffer Output buffer. + * @param surface Output surface. + * @throws FfmpegVideoDecoderException Thrown if called with invalid output mode or frame + * rendering fails. + */ + public void renderToSurface(VideoDecoderOutputBuffer outputBuffer, Surface surface) + throws FfmpegVideoDecoderException { + if (outputBuffer.mode != C.VIDEO_OUTPUT_MODE_SURFACE_YUV) { + throw new FfmpegVideoDecoderException("Invalid output mode."); + } + if (ffmpegRenderFrame( + nativeContext, surface, + outputBuffer, outputBuffer.width, outputBuffer.height) == DECODER_ERROR_OTHER) { + throw new FfmpegVideoDecoderException( + "Buffer render error: "); + } + } + + private native long ffmpegInitialize(String codecName, @Nullable byte[] extraData, int threads); + + private native long ffmpegReset(long context); + + private native void ffmpegRelease(long context); + + private native int ffmpegRenderFrame( + long context, Surface surface, VideoDecoderOutputBuffer outputBuffer, + int displayedWidth, + int displayedHeight); + + /** + * Decodes the encoded data passed. + * + * @param context Decoder context. + * @param encodedData Encoded data. + * @param length Length of the data buffer. + * @return 0 if successful, {@link #DECODER_ERROR_OTHER} if an error occurred. + */ + private native int ffmpegSendPacket(long context, ByteBuffer encodedData, int length, + long inputTime); + + /** + * Gets the decoded frame. + * + * @param context Decoder context. + * @param outputBuffer Output buffer for the decoded frame. + * @return 0 if successful, {@link #DECODER_ERROR_INVALID_DATA} if successful but the frame is + * decode-only, {@link #DECODER_ERROR_OTHER} if an error occurred. + */ + private native int ffmpegReceiveFrame( + long context, int outputMode, VideoDecoderOutputBuffer outputBuffer, boolean decodeOnly); + +} diff --git a/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegVideoDecoderException.java b/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegVideoDecoderException.java new file mode 100644 index 00000000000..164c183ea44 --- /dev/null +++ b/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegVideoDecoderException.java @@ -0,0 +1,30 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.ext.ffmpeg; + +import com.google.android.exoplayer2.video.VideoDecoderException; + +/** Thrown when a libgav1 decoder error occurs. */ +public final class FfmpegVideoDecoderException extends VideoDecoderException { + + /* package */ FfmpegVideoDecoderException(String message) { + super(message); + } + + /* package */ FfmpegVideoDecoderException(String message, Throwable cause) { + super(message, cause); + } +} diff --git a/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegVideoRenderer.java b/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegVideoRenderer.java new file mode 100644 index 00000000000..770a6a5fd21 --- /dev/null +++ b/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegVideoRenderer.java @@ -0,0 +1,190 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.google.android.exoplayer2.ext.ffmpeg; + +import static java.lang.Runtime.getRuntime; + +import android.os.Handler; +import android.view.Surface; +import androidx.annotation.Keep; +import androidx.annotation.Nullable; +import com.google.android.exoplayer2.C; +import com.google.android.exoplayer2.ExoPlaybackException; +import com.google.android.exoplayer2.ExoPlayer; +import com.google.android.exoplayer2.Format; +import com.google.android.exoplayer2.PlayerMessage.Target; +import com.google.android.exoplayer2.RendererCapabilities; +import com.google.android.exoplayer2.decoder.SimpleDecoder; +import com.google.android.exoplayer2.drm.ExoMediaCrypto; +import com.google.android.exoplayer2.util.TraceUtil; +import com.google.android.exoplayer2.util.Util; +import com.google.android.exoplayer2.video.SimpleDecoderVideoRenderer; +import com.google.android.exoplayer2.video.VideoDecoderException; +import com.google.android.exoplayer2.video.VideoDecoderInputBuffer; +import com.google.android.exoplayer2.video.VideoDecoderOutputBuffer; +import com.google.android.exoplayer2.video.VideoDecoderOutputBufferRenderer; +import com.google.android.exoplayer2.video.VideoRendererEventListener; + +/** + * Decodes and renders video using libgav1 decoder. + * + *

This renderer accepts the following messages sent via {@link ExoPlayer#createMessage(Target)} + * on the playback thread: + * + *

    + *
  • Message with type {@link #MSG_SET_SURFACE} to set the output surface. The message payload + * should be the target {@link Surface}, or null. + *
  • Message with type {@link #MSG_SET_VIDEO_DECODER_OUTPUT_BUFFER_RENDERER} to set the output + * buffer renderer. The message payload should be the target {@link + * VideoDecoderOutputBufferRenderer}, or null. + *
+ */ +@Keep +public class FfmpegVideoRenderer extends SimpleDecoderVideoRenderer { + + private static final int DEFAULT_NUM_OF_INPUT_BUFFERS = 4; + private static final int DEFAULT_NUM_OF_OUTPUT_BUFFERS = 4; + /* Default size based on 720p resolution video compressed by a factor of two. */ + private static final int DEFAULT_INPUT_BUFFER_SIZE = + Util.ceilDivide(1280, 64) * Util.ceilDivide(720, 64) * (64 * 64 * 3 / 2) / 2; + + /** The number of input buffers. */ + private final int numInputBuffers; + /** + * The number of output buffers. The renderer may limit the minimum possible value due to + * requiring multiple output buffers to be dequeued at a time for it to make progress. + */ + private final int numOutputBuffers; + + private final int threads; + + @Nullable private FfmpegVideoDecoder decoder; + + /** + * Creates a Libgav1VideoRenderer. + * + * @param allowedJoiningTimeMs The maximum duration in milliseconds for which this video renderer + * can attempt to seamlessly join an ongoing playback. + * @param eventHandler A handler to use when delivering events to {@code eventListener}. May be + * null if delivery of events is not required. + * @param eventListener A listener of events. May be null if delivery of events is not required. + * @param maxDroppedFramesToNotify The maximum number of frames that can be dropped between + * invocations of {@link VideoRendererEventListener#onDroppedFrames(int, long)}. + */ + public FfmpegVideoRenderer( + long allowedJoiningTimeMs, + @Nullable Handler eventHandler, + @Nullable VideoRendererEventListener eventListener, + int maxDroppedFramesToNotify) { + this( + allowedJoiningTimeMs, + eventHandler, + eventListener, + maxDroppedFramesToNotify, + /* threads= */ getRuntime().availableProcessors(), + DEFAULT_NUM_OF_INPUT_BUFFERS, + DEFAULT_NUM_OF_OUTPUT_BUFFERS); + } + + /** + * Creates a Libgav1VideoRenderer. + * + * @param allowedJoiningTimeMs The maximum duration in milliseconds for which this video renderer + * can attempt to seamlessly join an ongoing playback. + * @param eventHandler A handler to use when delivering events to {@code eventListener}. May be + * null if delivery of events is not required. + * @param eventListener A listener of events. May be null if delivery of events is not required. + * @param maxDroppedFramesToNotify The maximum number of frames that can be dropped between + * invocations of {@link VideoRendererEventListener#onDroppedFrames(int, long)}. + * @param threads Number of threads libgav1 will use to decode. + * @param numInputBuffers Number of input buffers. + * @param numOutputBuffers Number of output buffers. + */ + public FfmpegVideoRenderer( + long allowedJoiningTimeMs, + @Nullable Handler eventHandler, + @Nullable VideoRendererEventListener eventListener, + int maxDroppedFramesToNotify, + int threads, + int numInputBuffers, + int numOutputBuffers) { + super(allowedJoiningTimeMs, eventHandler, eventListener, maxDroppedFramesToNotify); + this.threads = threads; + this.numInputBuffers = numInputBuffers; + this.numOutputBuffers = numOutputBuffers; + } + + @Override + @Capabilities + public final int supportsFormat(Format format) { + if (!FfmpegLibrary.isAvailable() + || !FfmpegLibrary.supportsVideoFormat(format.sampleMimeType)) { + return RendererCapabilities.create(FORMAT_UNSUPPORTED_TYPE); + } + if (format.drmInitData != null && format.exoMediaCryptoType == null) { + return RendererCapabilities.create(FORMAT_UNSUPPORTED_DRM); + } + return RendererCapabilities.create(FORMAT_HANDLED, ADAPTIVE_SEAMLESS, TUNNELING_NOT_SUPPORTED); + } + + @Override + protected SimpleDecoder< + VideoDecoderInputBuffer, + ? extends VideoDecoderOutputBuffer, + ? extends VideoDecoderException> + createDecoder(Format format, @Nullable ExoMediaCrypto mediaCrypto) + throws VideoDecoderException { + TraceUtil.beginSection("createGav1Decoder"); + int initialInputBufferSize = + format.maxInputSize != Format.NO_VALUE ? format.maxInputSize : DEFAULT_INPUT_BUFFER_SIZE; + FfmpegVideoDecoder decoder = + new FfmpegVideoDecoder(numInputBuffers, numOutputBuffers, initialInputBufferSize, threads, format); + this.decoder = decoder; + TraceUtil.endSection(); + return decoder; + } + + @Override + protected void renderOutputBufferToSurface(VideoDecoderOutputBuffer outputBuffer, Surface surface) + throws FfmpegVideoDecoderException { + if (decoder == null) { + throw new FfmpegVideoDecoderException( + "Failed to render output buffer to surface: decoder is not initialized."); + } + decoder.renderToSurface(outputBuffer, surface); + outputBuffer.release(); + } + + @Override + protected void setDecoderOutputMode(@C.VideoOutputMode int outputMode) { + if (decoder != null) { + decoder.setOutputMode(outputMode); + } + } + + // PlayerMessage.Target implementation. + + @Override + public void handleMessage(int messageType, @Nullable Object message) throws ExoPlaybackException { + if (messageType == MSG_SET_SURFACE) { + setOutputSurface((Surface) message); + } else if (messageType == MSG_SET_VIDEO_DECODER_OUTPUT_BUFFER_RENDERER) { + setOutputBufferRenderer((VideoDecoderOutputBufferRenderer) message); + } else { + super.handleMessage(messageType, message); + } + } +} diff --git a/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/package-info.java b/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/package-info.java new file mode 100644 index 00000000000..a9fedb19cb6 --- /dev/null +++ b/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/package-info.java @@ -0,0 +1,19 @@ +/* + * Copyright (C) 2019 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@NonNullApi +package com.google.android.exoplayer2.ext.ffmpeg; + +import com.google.android.exoplayer2.util.NonNullApi; diff --git a/extensions/ffmpegvideo/src/main/jni/CMakeLists.txt b/extensions/ffmpegvideo/src/main/jni/CMakeLists.txt new file mode 100644 index 00000000000..de6720083f2 --- /dev/null +++ b/extensions/ffmpegvideo/src/main/jni/CMakeLists.txt @@ -0,0 +1,65 @@ +# libgav1JNI requires modern CMake. +cmake_minimum_required(VERSION 3.7.1 FATAL_ERROR) + +# libgav1JNI requires C++11. +set(CMAKE_CXX_STANDARD 11) + +project(libffmpegJNI C CXX) + +# Devices using armeabi-v7a are not required to support +# Neon which is why Neon is disabled by default for +# armeabi-v7a build. This flag enables it. +if(${ANDROID_ABI} MATCHES "armeabi-v7a") + add_compile_options("-mfpu=neon") + add_compile_options("-fPIC") +endif() + +set(libgav1_jni_root "${CMAKE_CURRENT_SOURCE_DIR}") +set(libgav1_jni_build "${CMAKE_BINARY_DIR}") +set(libgav1_jni_output_directory + ${libgav1_jni_root}/../libs/${ANDROID_ABI}/) + +#set(libgav1_root "${libgav1_jni_root}/libgav1") +#set(libgav1_build "${libgav1_jni_build}/libgav1") + +#set(cpu_features_root "${libgav1_jni_root}/cpu_features") +#set(cpu_features_build "${libgav1_jni_build}/cpu_features") + +# Build cpu_features library. +#add_subdirectory("${cpu_features_root}" +# "${cpu_features_build}" +# EXCLUDE_FROM_ALL) + +# Build libgav1. +#add_subdirectory("${libgav1_root}" +# "${libgav1_build}" +# EXCLUDE_FROM_ALL) +add_library(ffmpeg + SHARED + IMPORTED) +set_target_properties(ffmpeg PROPERTIES + IMPORTED_LOCATION + ${libgav1_jni_output_directory}/libffmpeg.so) + +# Build libgav1JNI. +add_library(ffmpegJNI + SHARED + ffmpeg_audio_jni.cc + ffmpeg_video_jni.cc) +include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include) + +# Locate NDK log library. +find_library(android_log_lib log) + +# Link libgav1JNI against used libraries. +target_link_libraries(ffmpegJNI + PRIVATE android + PRIVATE ffmpeg +# PRIVATE cpu_features +# PRIVATE libgav1_static + PRIVATE ${android_log_lib}) + +# Specify output directory for libgav1JNI. +set_target_properties(ffmpegJNI PROPERTIES + LIBRARY_OUTPUT_DIRECTORY + ${libgav1_jni_output_directory}) diff --git a/extensions/ffmpegvideo/src/main/jni/ffmpeg_audio_jni.cc b/extensions/ffmpegvideo/src/main/jni/ffmpeg_audio_jni.cc new file mode 100644 index 00000000000..89dc2d3de06 --- /dev/null +++ b/extensions/ffmpegvideo/src/main/jni/ffmpeg_audio_jni.cc @@ -0,0 +1,360 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include + +extern "C" { +#ifdef __cplusplus +#define __STDC_CONSTANT_MACROS +#ifdef _STDINT_H +#undef _STDINT_H +#endif +#include +#endif +#include +#include +#include +#include +#include +} + +#define LOG_TAG "ffmpeg_jni" +#define LOGE(...) ((void)__android_log_print(ANDROID_LOG_ERROR, LOG_TAG, \ + __VA_ARGS__)) + +#define DECODER_FUNC(RETURN_TYPE, NAME, ...) \ + extern "C" { \ + JNIEXPORT RETURN_TYPE \ + Java_com_google_android_exoplayer2_ext_ffmpeg_FfmpegAudioDecoder_ ## NAME \ + (JNIEnv* env, jobject thiz, ##__VA_ARGS__);\ + } \ + JNIEXPORT RETURN_TYPE \ + Java_com_google_android_exoplayer2_ext_ffmpeg_FfmpegAudioDecoder_ ## NAME \ + (JNIEnv* env, jobject thiz, ##__VA_ARGS__)\ + +#define LIBRARY_FUNC(RETURN_TYPE, NAME, ...) \ + extern "C" { \ + JNIEXPORT RETURN_TYPE \ + Java_com_google_android_exoplayer2_ext_ffmpeg_FfmpegLibrary_ ## NAME \ + (JNIEnv* env, jobject thiz, ##__VA_ARGS__);\ + } \ + JNIEXPORT RETURN_TYPE \ + Java_com_google_android_exoplayer2_ext_ffmpeg_FfmpegLibrary_ ## NAME \ + (JNIEnv* env, jobject thiz, ##__VA_ARGS__)\ + +#define ERROR_STRING_BUFFER_LENGTH 256 + +// Output format corresponding to AudioFormat.ENCODING_PCM_16BIT. +static const AVSampleFormat OUTPUT_FORMAT_PCM_16BIT = AV_SAMPLE_FMT_S16; +// Output format corresponding to AudioFormat.ENCODING_PCM_FLOAT. +static const AVSampleFormat OUTPUT_FORMAT_PCM_FLOAT = AV_SAMPLE_FMT_FLT; + +// Error codes matching FfmpegAudioDecoder.java. +static const int DECODER_ERROR_INVALID_DATA = -1; +static const int DECODER_ERROR_OTHER = -2; + +/** + * Returns the AVCodec with the specified name, or NULL if it is not available. + */ +AVCodec *getCodecByName(JNIEnv* env, jstring codecName); + +/** + * Allocates and opens a new AVCodecContext for the specified codec, passing the + * provided extraData as initialization data for the decoder if it is non-NULL. + * Returns the created context. + */ +AVCodecContext *createContext(JNIEnv *env, AVCodec *codec, jbyteArray extraData, + jboolean outputFloat, jint rawSampleRate, + jint rawChannelCount); + +/** + * Decodes the packet into the output buffer, returning the number of bytes + * written, or a negative DECODER_ERROR constant value in the case of an error. + */ +int decodePacket(AVCodecContext *context, AVPacket *packet, + uint8_t *outputBuffer, int outputSize); + +/** + * Outputs a log message describing the avcodec error number. + */ +void logError(const char *functionName, int errorNumber); + +/** + * Releases the specified context. + */ +void releaseContext(AVCodecContext *context); + +jint JNI_OnLoad(JavaVM *vm, void *reserved) { + JNIEnv *env; + if (vm->GetEnv(reinterpret_cast(&env), JNI_VERSION_1_6) != JNI_OK) { + return -1; + } + avcodec_register_all(); + return JNI_VERSION_1_6; +} + +LIBRARY_FUNC(jstring, ffmpegGetVersion) { + return env->NewStringUTF(LIBAVCODEC_IDENT); +} + +LIBRARY_FUNC(jboolean, ffmpegHasDecoder, jstring codecName) { + return getCodecByName(env, codecName) != NULL; +} + +DECODER_FUNC(jlong, ffmpegInitialize, jstring codecName, jbyteArray extraData, + jboolean outputFloat, jint rawSampleRate, jint rawChannelCount) { + AVCodec *codec = getCodecByName(env, codecName); + if (!codec) { + LOGE("Codec not found."); + return 0L; + } + return (jlong)createContext(env, codec, extraData, outputFloat, rawSampleRate, + rawChannelCount); +} + +DECODER_FUNC(jint, ffmpegDecode, jlong context, jobject inputData, + jint inputSize, jobject outputData, jint outputSize) { + if (!context) { + LOGE("Context must be non-NULL."); + return -1; + } + if (!inputData || !outputData) { + LOGE("Input and output buffers must be non-NULL."); + return -1; + } + if (inputSize < 0) { + LOGE("Invalid input buffer size: %d.", inputSize); + return -1; + } + if (outputSize < 0) { + LOGE("Invalid output buffer length: %d", outputSize); + return -1; + } + uint8_t *inputBuffer = (uint8_t *) env->GetDirectBufferAddress(inputData); + uint8_t *outputBuffer = (uint8_t *) env->GetDirectBufferAddress(outputData); + AVPacket packet; + av_init_packet(&packet); + packet.data = inputBuffer; + packet.size = inputSize; + return decodePacket((AVCodecContext *) context, &packet, outputBuffer, + outputSize); +} + +DECODER_FUNC(jint, ffmpegGetChannelCount, jlong context) { + if (!context) { + LOGE("Context must be non-NULL."); + return -1; + } + return ((AVCodecContext *) context)->channels; +} + +DECODER_FUNC(jint, ffmpegGetSampleRate, jlong context) { + if (!context) { + LOGE("Context must be non-NULL."); + return -1; + } + return ((AVCodecContext *) context)->sample_rate; +} + +DECODER_FUNC(jlong, ffmpegReset, jlong jContext, jbyteArray extraData) { + AVCodecContext *context = (AVCodecContext *) jContext; + if (!context) { + LOGE("Tried to reset without a context."); + return 0L; + } + + AVCodecID codecId = context->codec_id; + if (codecId == AV_CODEC_ID_TRUEHD) { + // Release and recreate the context if the codec is TrueHD. + // TODO: Figure out why flushing doesn't work for this codec. + releaseContext(context); + AVCodec *codec = avcodec_find_decoder(codecId); + if (!codec) { + LOGE("Unexpected error finding codec %d.", codecId); + return 0L; + } + jboolean outputFloat = + (jboolean)(context->request_sample_fmt == OUTPUT_FORMAT_PCM_FLOAT); + return (jlong)createContext(env, codec, extraData, outputFloat, + /* rawSampleRate= */ -1, + /* rawChannelCount= */ -1); + } + + avcodec_flush_buffers(context); + return (jlong) context; +} + +DECODER_FUNC(void, ffmpegRelease, jlong context) { + if (context) { + releaseContext((AVCodecContext *) context); + } +} + +AVCodec *getCodecByName(JNIEnv* env, jstring codecName) { + if (!codecName) { + return NULL; + } + const char *codecNameChars = env->GetStringUTFChars(codecName, NULL); + AVCodec *codec = avcodec_find_decoder_by_name(codecNameChars); + env->ReleaseStringUTFChars(codecName, codecNameChars); + return codec; +} + +AVCodecContext *createContext(JNIEnv *env, AVCodec *codec, jbyteArray extraData, + jboolean outputFloat, jint rawSampleRate, + jint rawChannelCount) { + AVCodecContext *context = avcodec_alloc_context3(codec); + if (!context) { + LOGE("Failed to allocate context."); + return NULL; + } + context->request_sample_fmt = + outputFloat ? OUTPUT_FORMAT_PCM_FLOAT : OUTPUT_FORMAT_PCM_16BIT; + if (extraData) { + jsize size = env->GetArrayLength(extraData); + context->extradata_size = size; + context->extradata = + (uint8_t *) av_malloc(size + AV_INPUT_BUFFER_PADDING_SIZE); + if (!context->extradata) { + LOGE("Failed to allocate extradata."); + releaseContext(context); + return NULL; + } + env->GetByteArrayRegion(extraData, 0, size, (jbyte *) context->extradata); + } + if (context->codec_id == AV_CODEC_ID_PCM_MULAW || + context->codec_id == AV_CODEC_ID_PCM_ALAW) { + context->sample_rate = rawSampleRate; + context->channels = rawChannelCount; + context->channel_layout = av_get_default_channel_layout(rawChannelCount); + } + context->err_recognition = AV_EF_IGNORE_ERR; + int result = avcodec_open2(context, codec, NULL); + if (result < 0) { + logError("avcodec_open2", result); + releaseContext(context); + return NULL; + } + return context; +} + +int decodePacket(AVCodecContext *context, AVPacket *packet, + uint8_t *outputBuffer, int outputSize) { + int result = 0; + // Queue input data. + result = avcodec_send_packet(context, packet); + if (result) { + logError("avcodec_send_packet", result); + return result == AVERROR_INVALIDDATA ? DECODER_ERROR_INVALID_DATA + : DECODER_ERROR_OTHER; + } + + // Dequeue output data until it runs out. + int outSize = 0; + while (true) { + AVFrame *frame = av_frame_alloc(); + if (!frame) { + LOGE("Failed to allocate output frame."); + return -1; + } + result = avcodec_receive_frame(context, frame); + if (result) { + av_frame_free(&frame); + if (result == AVERROR(EAGAIN)) { + break; + } + logError("avcodec_receive_frame", result); + return result; + } + + // Resample output. + AVSampleFormat sampleFormat = context->sample_fmt; + int channelCount = context->channels; + int channelLayout = context->channel_layout; + int sampleRate = context->sample_rate; + int sampleCount = frame->nb_samples; + int dataSize = av_samples_get_buffer_size(NULL, channelCount, sampleCount, + sampleFormat, 1); + SwrContext *resampleContext; + if (context->opaque) { + resampleContext = (SwrContext *)context->opaque; + } else { + resampleContext = swr_alloc(); + av_opt_set_int(resampleContext, "in_channel_layout", channelLayout, 0); + av_opt_set_int(resampleContext, "out_channel_layout", channelLayout, 0); + av_opt_set_int(resampleContext, "in_sample_rate", sampleRate, 0); + av_opt_set_int(resampleContext, "out_sample_rate", sampleRate, 0); + av_opt_set_int(resampleContext, "in_sample_fmt", sampleFormat, 0); + // The output format is always the requested format. + av_opt_set_int(resampleContext, "out_sample_fmt", + context->request_sample_fmt, 0); + result = swr_init(resampleContext); + if (result < 0) { + logError("swr_init", result); + av_frame_free(&frame); + return -1; + } + context->opaque = resampleContext; + } + int inSampleSize = av_get_bytes_per_sample(sampleFormat); + int outSampleSize = av_get_bytes_per_sample(context->request_sample_fmt); + int outSamples = swr_get_out_samples(resampleContext, sampleCount); + int bufferOutSize = outSampleSize * channelCount * outSamples; + if (outSize + bufferOutSize > outputSize) { + LOGE("Output buffer size (%d) too small for output data (%d).", + outputSize, outSize + bufferOutSize); + av_frame_free(&frame); + return -1; + } + result = swr_convert(resampleContext, &outputBuffer, bufferOutSize, + (const uint8_t **)frame->data, frame->nb_samples); + av_frame_free(&frame); + if (result < 0) { + logError("swr_convert", result); + return result; + } + int available = swr_get_out_samples(resampleContext, 0); + if (available != 0) { + LOGE("Expected no samples remaining after resampling, but found %d.", + available); + return -1; + } + outputBuffer += bufferOutSize; + outSize += bufferOutSize; + } + return outSize; +} + +void logError(const char *functionName, int errorNumber) { + char *buffer = (char *) malloc(ERROR_STRING_BUFFER_LENGTH * sizeof(char)); + av_strerror(errorNumber, buffer, ERROR_STRING_BUFFER_LENGTH); + LOGE("Error in %s: %s", functionName, buffer); + free(buffer); +} + +void releaseContext(AVCodecContext *context) { + if (!context) { + return; + } + SwrContext *swrContext; + if ((swrContext = (SwrContext *)context->opaque)) { + swr_free(&swrContext); + context->opaque = NULL; + } + avcodec_free_context(&context); +} + diff --git a/extensions/ffmpegvideo/src/main/jni/ffmpeg_video_jni.cc b/extensions/ffmpegvideo/src/main/jni/ffmpeg_video_jni.cc new file mode 100644 index 00000000000..2cb5a1c16d9 --- /dev/null +++ b/extensions/ffmpegvideo/src/main/jni/ffmpeg_video_jni.cc @@ -0,0 +1,436 @@ +/* + * Copyright (C) 2016 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +#include +#include +#include +#include +#include +#include + +extern "C" { +#ifdef __cplusplus +#define __STDC_CONSTANT_MACROS +#ifdef _STDINT_H +#undef _STDINT_H +#endif + +#endif +#include +#include +#include +#include +#include +} + +#define LOG_TAG "ffmpeg_jni" +#define LOGE(...) ((void)__android_log_print(ANDROID_LOG_ERROR, LOG_TAG, \ + __VA_ARGS__)) + +#define DECODER_FUNC(RETURN_TYPE, NAME, ...) \ + extern "C" { \ + JNIEXPORT RETURN_TYPE \ + Java_com_google_android_exoplayer2_ext_ffmpeg_FfmpegVideoDecoder_ ## NAME \ + (JNIEnv* env, jobject thiz, ##__VA_ARGS__);\ + } \ + JNIEXPORT RETURN_TYPE \ + Java_com_google_android_exoplayer2_ext_ffmpeg_FfmpegVideoDecoder_ ## NAME \ + (JNIEnv* env, jobject thiz, ##__VA_ARGS__)\ + +#define ERROR_STRING_BUFFER_LENGTH 256 + + +namespace { +// Error codes matching FfmpegAudioDecoder.java. +const int DECODER_SUCCESS = 0; +const int DECODER_ERROR_INVALID_DATA = -1; +const int DECODER_ERROR_OTHER = -2; +const int DECODER_ERROR_READ_FRAME = -3; +const int DECODER_ERROR_SEND_PACKET = -4; + +// YUV plane indices. +const int kPlaneY = 0; +const int kPlaneU = 1; +const int kPlaneV = 2; +const int kMaxPlanes = 3; + +// Android YUV format. See: +// https://developer.android.com/reference/android/graphics/ImageFormat.html#YV12. +const int kImageFormatYV12 = 0x32315659; + +// LINT.IfChange +// Output modes. +const int kOutputModeYuv = 0; +const int kOutputModeSurfaceYuv = 1; +// LINT.ThenChange(../../../../../library/core/src/main/java/com/google/android/exoplayer2/C.java) + +// LINT.IfChange +const int kColorSpaceUnknown = 0; +// LINT.ThenChange(../../../../../library/core/src/main/java/com/google/android/exoplayer2/video/VideoDecoderOutputBuffer.java) + +struct JniContext { + ~JniContext() { + if (native_window) { + ANativeWindow_release(native_window); + } + } + + bool MaybeAcquireNativeWindow(JNIEnv *env, jobject new_surface) { + if (surface == new_surface) { + return true; + } + if (native_window) { + ANativeWindow_release(native_window); + } + native_window_width = 0; + native_window_height = 0; + native_window = ANativeWindow_fromSurface(env, new_surface); + if (native_window == nullptr) { + LOGE("kJniStatusANativeWindowError"); + surface = nullptr; + return false; + } + surface = new_surface; + return true; + } + + jfieldID data_field; + jfieldID yuvPlanes_field; + jfieldID yuvStrides_field; + jmethodID init_for_private_frame_method; + jmethodID init_for_yuv_frame_method; + jmethodID init_method; + + AVCodecContext *codecContext; + + ANativeWindow *native_window = nullptr; + jobject surface = nullptr; + int native_window_width = 0; + int native_window_height = 0; +}; + + +AVCodec *getCodecByName(JNIEnv *env, jstring codecName) { + if (!codecName) { + return NULL; + } + const char *codecNameChars = env->GetStringUTFChars(codecName, NULL); + AVCodec *codec = avcodec_find_decoder_by_name(codecNameChars); + env->ReleaseStringUTFChars(codecName, codecNameChars); + return codec; +} + +void logError(const char *functionName, int errorNumber) { + char *buffer = (char *) malloc(ERROR_STRING_BUFFER_LENGTH * sizeof(char)); + av_strerror(errorNumber, buffer, ERROR_STRING_BUFFER_LENGTH); + LOGE("Error in %s: %s", functionName, buffer); + free(buffer); +} + +void releaseContext(AVCodecContext *context) { + if (!context) { + return; + } + + avcodec_free_context(&context); +} + +JniContext *createContext(JNIEnv *env, + AVCodec *codec, + jbyteArray extraData, + jint threads) { + JniContext *jniContext = new(std::nothrow) JniContext(); + + AVCodecContext *codecContext = avcodec_alloc_context3(codec); + if (!codecContext) { + LOGE("Failed to allocate context."); + return NULL; + } + + if (extraData) { + jsize size = env->GetArrayLength(extraData); + codecContext->extradata_size = size; + codecContext->extradata = + (uint8_t *) av_malloc(size + AV_INPUT_BUFFER_PADDING_SIZE); + if (!codecContext->extradata) { + LOGE("Failed to allocate extradata."); + releaseContext(codecContext); + return NULL; + } + env->GetByteArrayRegion(extraData, 0, size, (jbyte *) codecContext->extradata); + } + + codecContext->thread_count = threads; + codecContext->err_recognition = AV_EF_IGNORE_ERR; + int result = avcodec_open2(codecContext, codec, NULL); + if (result < 0) { + logError("avcodec_open2", result); + releaseContext(codecContext); + return NULL; + } + + jniContext->codecContext = codecContext; + + // Populate JNI References. + const jclass outputBufferClass = env->FindClass( + "com/google/android/exoplayer2/video/VideoDecoderOutputBuffer"); + jniContext->data_field = env->GetFieldID(outputBufferClass, "data", "Ljava/nio/ByteBuffer;"); + jniContext->yuvPlanes_field = + env->GetFieldID(outputBufferClass, "yuvPlanes", "[Ljava/nio/ByteBuffer;"); + jniContext->yuvStrides_field = env->GetFieldID(outputBufferClass, "yuvStrides", "[I"); + jniContext->init_for_private_frame_method = + env->GetMethodID(outputBufferClass, "initForPrivateFrame", "(II)V"); + jniContext->init_for_yuv_frame_method = + env->GetMethodID(outputBufferClass, "initForYuvFrame", "(IIIII)Z"); + jniContext->init_method = + env->GetMethodID(outputBufferClass, "init", "(JILjava/nio/ByteBuffer;)V"); + + return jniContext; +} + +void CopyPlane(const uint8_t *source, int source_stride, uint8_t *destination, + int destination_stride, int width, int height) { + while (height--) { + std::memcpy(destination, source, width); + source += source_stride; + destination += destination_stride; + } +} + +constexpr int AlignTo16(int value) { return (value + 15) & (~15); } + +} + +DECODER_FUNC(jlong, ffmpegInitialize, jstring codecName, jbyteArray extraData, jint threads) { + AVCodec *codec = getCodecByName(env, codecName); + if (!codec) { + LOGE("Codec not found."); + return 0L; + } + + return (jlong) createContext(env, codec, extraData, threads); +} + +DECODER_FUNC(jlong, ffmpegReset, jlong jContext) { + JniContext *const jniContext = reinterpret_cast(jContext); + AVCodecContext *context = jniContext->codecContext; + if (!context) { + LOGE("Tried to reset without a context."); + return 0L; + } + +// LOGE("avcodec_flush_buffers"); + avcodec_flush_buffers(context); + return (jlong) jniContext; +} + +DECODER_FUNC(void, ffmpegRelease, jlong jContext) { + JniContext *const jniContext = reinterpret_cast(jContext); + AVCodecContext *context = jniContext->codecContext; + if (context) { + releaseContext(context); + } +} + + +DECODER_FUNC(jint, ffmpegSendPacket, jlong jContext, jobject encodedData, + jint length, jlong inputTimeUs) { + JniContext *const jniContext = reinterpret_cast(jContext); + AVCodecContext *avContext = jniContext->codecContext; + + uint8_t *inputBuffer = (uint8_t *) env->GetDirectBufferAddress(encodedData); + AVPacket packet; + av_init_packet(&packet); + packet.data = inputBuffer; + packet.size = length; + packet.pts = inputTimeUs; + + int result = 0; + // Queue input data. + result = avcodec_send_packet(avContext, &packet); + if (result) { + logError("avcodec_send_packet", result); + if (result == AVERROR_INVALIDDATA) { + // need more data + return DECODER_ERROR_INVALID_DATA; + } else if (result == AVERROR(EAGAIN)) { + // need read frame + return DECODER_ERROR_READ_FRAME; + } else { + return DECODER_ERROR_OTHER; + } + } + return result; +} + +DECODER_FUNC(jint, ffmpegReceiveFrame, jlong jContext, jint outputMode, jobject jOutputBuffer, + jboolean decodeOnly) { + JniContext *const jniContext = reinterpret_cast(jContext); + AVCodecContext *avContext = jniContext->codecContext; + int result = 0; + + AVFrame *frame = av_frame_alloc(); + if (!frame) { + LOGE("Failed to allocate output frame."); + return DECODER_ERROR_OTHER; + } + result = avcodec_receive_frame(avContext, frame); + + // fail + if (decodeOnly || result == AVERROR(EAGAIN)) { + // This is not an error. The input data was decode-only or no displayable + // frames are available. + av_frame_free(&frame); + return DECODER_ERROR_INVALID_DATA; + } + if (result) { + av_frame_free(&frame); + logError("avcodec_receive_frame", result); + return DECODER_ERROR_OTHER; + } + + // success + // init time and mode + env->CallVoidMethod(jOutputBuffer, jniContext->init_method, frame->pts, outputMode, nullptr); + + // init data + const jboolean init_result = env->CallBooleanMethod( + jOutputBuffer, jniContext->init_for_yuv_frame_method, + frame->width, + frame->height, + frame->linesize[0], frame->linesize[1], + 0); + if (env->ExceptionCheck()) { + // Exception is thrown in Java when returning from the native call. + return DECODER_ERROR_OTHER; + } + if (!init_result) { + return DECODER_ERROR_OTHER; + } + + const jobject data_object = env->GetObjectField(jOutputBuffer, jniContext->data_field); + jbyte *data = reinterpret_cast(env->GetDirectBufferAddress(data_object)); + const int32_t uvHeight = (frame->height + 1) / 2; + const uint64_t yLength = frame->linesize[0] * frame->height; + const uint64_t uvLength = frame->linesize[1] * uvHeight; + + // todo rotate YUV data + + memcpy(data, frame->data[0], yLength); + memcpy(data + yLength, frame->data[1], uvLength); + memcpy(data + yLength + uvLength, frame->data[2], uvLength); + + av_frame_free(&frame); + + return result; +} + +DECODER_FUNC(jint, ffmpegRenderFrame, jlong jContext, jobject jSurface, + jobject jOutputBuffer, jint displayedWidth, jint displayedHeight) { + JniContext *const jniContext = reinterpret_cast(jContext); + if (!jniContext->MaybeAcquireNativeWindow(env, jSurface)) { + return DECODER_ERROR_OTHER; + } + + if (jniContext->native_window_width != displayedWidth || + jniContext->native_window_height != displayedHeight) { + if (ANativeWindow_setBuffersGeometry( + jniContext->native_window, + displayedWidth, + displayedHeight, + kImageFormatYV12)) { + LOGE("kJniStatusANativeWindowError"); + return DECODER_ERROR_OTHER; + } + jniContext->native_window_width = displayedWidth; + jniContext->native_window_height = displayedHeight; + } + + ANativeWindow_Buffer native_window_buffer; + if (ANativeWindow_lock(jniContext->native_window, &native_window_buffer, + /*inOutDirtyBounds=*/nullptr) || + native_window_buffer.bits == nullptr) { + LOGE("kJniStatusANativeWindowError"); + return DECODER_ERROR_OTHER; + } + + jobject yuvPlanes_object = env->GetObjectField(jOutputBuffer, jniContext->yuvPlanes_field); + jobjectArray yuvPlanes_array = static_cast(yuvPlanes_object); + jobject yuvPlanesY = env->GetObjectArrayElement(yuvPlanes_array, kPlaneY); + jobject yuvPlanesU = env->GetObjectArrayElement(yuvPlanes_array, kPlaneU); + jobject yuvPlanesV = env->GetObjectArrayElement(yuvPlanes_array, kPlaneV); + jbyte *planeY = reinterpret_cast(env->GetDirectBufferAddress(yuvPlanesY)); + jbyte *planeU = reinterpret_cast(env->GetDirectBufferAddress(yuvPlanesU)); + jbyte *planeV = reinterpret_cast(env->GetDirectBufferAddress(yuvPlanesV)); + + jobject yuvStrides_object = env->GetObjectField(jOutputBuffer, jniContext->yuvStrides_field); + jintArray *yuvStrides_array = reinterpret_cast(&yuvStrides_object); + + int *yuvStrides = env->GetIntArrayElements(*yuvStrides_array, NULL); + int strideY = yuvStrides[kPlaneY]; + int strideU = yuvStrides[kPlaneU]; + int strideV = yuvStrides[kPlaneV]; + + // Y plane + CopyPlane(reinterpret_cast(planeY), + strideY, + reinterpret_cast(native_window_buffer.bits), + native_window_buffer.stride, + displayedWidth, + displayedHeight); + + const int y_plane_size = + native_window_buffer.stride * native_window_buffer.height; + const int32_t native_window_buffer_uv_height = + (native_window_buffer.height + 1) / 2; + const int native_window_buffer_uv_stride = + AlignTo16(native_window_buffer.stride / 2); + + // TODO(b/140606738): Handle monochrome videos. + + // V plane + // Since the format for ANativeWindow is YV12, V plane is being processed + // before U plane. + const int v_plane_height = std::min(native_window_buffer_uv_height, + displayedHeight); + CopyPlane( + reinterpret_cast(planeV), + strideV, + reinterpret_cast(native_window_buffer.bits) + y_plane_size, + native_window_buffer_uv_stride, displayedWidth, + v_plane_height); + + const int v_plane_size = v_plane_height * native_window_buffer_uv_stride; + + // U plane + CopyPlane( + reinterpret_cast(planeU), + strideU, + reinterpret_cast(native_window_buffer.bits) + + y_plane_size + v_plane_size, + native_window_buffer_uv_stride, displayedWidth, + std::min(native_window_buffer_uv_height, + displayedHeight)); + + + env->ReleaseIntArrayElements(*yuvStrides_array, yuvStrides, 0); + + if (ANativeWindow_unlockAndPost(jniContext->native_window)) { + LOGE("kJniStatusANativeWindowError"); + return DECODER_ERROR_OTHER; + } + + return DECODER_SUCCESS; +} From 4497919cf2f83e55c0deb19ca8fce0e800f4cb8c Mon Sep 17 00:00:00 2001 From: haohao <358297604@qq.com> Date: Sat, 21 Mar 2020 02:10:41 +0800 Subject: [PATCH 2/5] Merge ffmpeg video renderer to current ffmpeg extension --- .gitignore | 4 +- core_settings.gradle | 2 - extensions/ffmpeg/build.gradle | 36 ++ .../ext/ffmpeg/FfmpegAudioDecoder.java | 27 +- .../ext/ffmpeg/FfmpegAudioRenderer.java | 6 +- .../exoplayer2/ext/ffmpeg/FfmpegDecoder.java | 229 --------- .../exoplayer2/ext/ffmpeg/FfmpegLibrary.java | 4 +- .../ext/ffmpeg/FfmpegVideoDecoder.java | 44 +- .../ext/ffmpeg/FfmpegVideoRenderer.java | 86 +++- .../src/main/jni/CMakeLists.txt | 37 +- extensions/ffmpeg/src/main/jni/ffmpeg_jni.cc | 382 ++++++++++++++- extensions/ffmpegvideo/README.md | 86 ---- extensions/ffmpegvideo/build.gradle | 90 ---- extensions/ffmpegvideo/proguard-rules.txt | 7 - .../ffmpegvideo/src/main/AndroidManifest.xml | 17 - .../ext/ffmpeg/FFmpegRenderersFactory.java | 137 ------ .../ffmpeg/FfmpegAudioDecoderException.java | 32 -- .../ext/ffmpeg/FfmpegAudioRenderer.java | 157 ------- .../exoplayer2/ext/ffmpeg/FfmpegLibrary.java | 164 ------- .../ffmpeg/FfmpegVideoDecoderException.java | 30 -- .../ext/ffmpeg/FfmpegVideoRenderer.java | 190 -------- .../exoplayer2/ext/ffmpeg/package-info.java | 19 - .../src/main/jni/ffmpeg_audio_jni.cc | 360 --------------- .../src/main/jni/ffmpeg_video_jni.cc | 436 ------------------ 24 files changed, 537 insertions(+), 2045 deletions(-) rename extensions/{ffmpegvideo => ffmpeg}/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegAudioDecoder.java (89%) delete mode 100644 extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegDecoder.java rename extensions/{ffmpegvideo => ffmpeg}/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegVideoDecoder.java (82%) rename extensions/{ffmpegvideo => ffmpeg}/src/main/jni/CMakeLists.txt (58%) delete mode 100644 extensions/ffmpegvideo/README.md delete mode 100644 extensions/ffmpegvideo/build.gradle delete mode 100644 extensions/ffmpegvideo/proguard-rules.txt delete mode 100644 extensions/ffmpegvideo/src/main/AndroidManifest.xml delete mode 100644 extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FFmpegRenderersFactory.java delete mode 100644 extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegAudioDecoderException.java delete mode 100644 extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegAudioRenderer.java delete mode 100644 extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegLibrary.java delete mode 100644 extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegVideoDecoderException.java delete mode 100644 extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegVideoRenderer.java delete mode 100644 extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/package-info.java delete mode 100644 extensions/ffmpegvideo/src/main/jni/ffmpeg_audio_jni.cc delete mode 100644 extensions/ffmpegvideo/src/main/jni/ffmpeg_video_jni.cc diff --git a/.gitignore b/.gitignore index 4c85a0f69f7..53de1451b32 100644 --- a/.gitignore +++ b/.gitignore @@ -69,8 +69,8 @@ extensions/flac/src/main/jni/flac # FFmpeg extension extensions/ffmpeg/src/main/jni/ffmpeg -extensions/ffmpegvideo/.cxx -extensions/ffmpegvideo/src/main/jni/include +extensions/ffmpeg/.cxx +extensions/ffmpeg/src/main/jni/include # Cronet extension extensions/cronet/jniLibs/* diff --git a/core_settings.gradle b/core_settings.gradle index 3f7e9956726..ac569331556 100644 --- a/core_settings.gradle +++ b/core_settings.gradle @@ -29,7 +29,6 @@ include modulePrefix + 'testutils' include modulePrefix + 'testdata' include modulePrefix + 'extension-av1' include modulePrefix + 'extension-ffmpeg' -include modulePrefix + 'extension-ffmpegvideo' include modulePrefix + 'extension-flac' include modulePrefix + 'extension-gvr' include modulePrefix + 'extension-ima' @@ -56,7 +55,6 @@ project(modulePrefix + 'testutils').projectDir = new File(rootDir, 'testutils') project(modulePrefix + 'testdata').projectDir = new File(rootDir, 'testdata') project(modulePrefix + 'extension-av1').projectDir = new File(rootDir, 'extensions/av1') project(modulePrefix + 'extension-ffmpeg').projectDir = new File(rootDir, 'extensions/ffmpeg') -project(modulePrefix + 'extension-ffmpegvideo').projectDir = new File(rootDir, 'extensions/ffmpegvideo') project(modulePrefix + 'extension-flac').projectDir = new File(rootDir, 'extensions/flac') project(modulePrefix + 'extension-gvr').projectDir = new File(rootDir, 'extensions/gvr') project(modulePrefix + 'extension-ima').projectDir = new File(rootDir, 'extensions/ima') diff --git a/extensions/ffmpeg/build.gradle b/extensions/ffmpeg/build.gradle index 26a72ae3357..83b653f06e9 100644 --- a/extensions/ffmpeg/build.gradle +++ b/extensions/ffmpeg/build.gradle @@ -26,9 +26,45 @@ android { minSdkVersion project.ext.minSdkVersion targetSdkVersion project.ext.targetSdkVersion consumerProguardFiles 'proguard-rules.txt' + + externalNativeBuild { + cmake { + // Debug CMake build type causes video frames to drop, + // so native library should always use Release build type. + arguments "-DCMAKE_BUILD_TYPE=Release" + targets "ffmpegJNI" + } + } + } + + externalNativeBuild { + cmake { + version '3.10.2' + path "src/main/jni/CMakeLists.txt" + } + } + + buildTypes { + debug { + ndk { + abiFilters 'arm64-v8a'/*, 'x86_64'*/ + } + } + } + + // This option resolves the problem of finding libffmpegJNI.so + // on multiple paths. The first one found is picked. + packagingOptions { + pickFirst 'lib/arm64-v8a/libffmpegJNI.so' + pickFirst 'lib/armeabi-v7a/libffmpegJNI.so' + pickFirst 'lib/x86/libffmpegJNI.so' + pickFirst 'lib/x86_64/libffmpegJNI.so' } sourceSets.main { + // As native JNI library build is invoked from gradle, this is + // not needed. However, it exposes the built library and keeps + // consistency with the other extensions. jniLibs.srcDir 'src/main/libs' jni.srcDirs = [] // Disable the automatic ndk-build call by Android Studio. } diff --git a/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegAudioDecoder.java b/extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegAudioDecoder.java similarity index 89% rename from extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegAudioDecoder.java rename to extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegAudioDecoder.java index f0ac07b8aa7..ca957bd0db2 100644 --- a/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegAudioDecoder.java +++ b/extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegAudioDecoder.java @@ -32,13 +32,13 @@ * FFmpeg audio decoder. */ /* package */ final class FfmpegAudioDecoder extends - SimpleDecoder { + SimpleDecoder { // Output buffer sizes when decoding PCM mu-law streams, which is the maximum FFmpeg outputs. private static final int OUTPUT_BUFFER_SIZE_16BIT = 65536; private static final int OUTPUT_BUFFER_SIZE_32BIT = OUTPUT_BUFFER_SIZE_16BIT * 2; - // Error codes matching ffmpeg_audio_jni.cc. + // Error codes matching ffmpeg_jni.cc. private static final int DECODER_ERROR_INVALID_DATA = -1; private static final int DECODER_ERROR_OTHER = -2; @@ -58,21 +58,20 @@ public FfmpegAudioDecoder( int initialInputBufferSize, Format format, boolean outputFloat) - throws FfmpegAudioDecoderException { + throws FfmpegDecoderException { super(new DecoderInputBuffer[numInputBuffers], new SimpleOutputBuffer[numOutputBuffers]); if (!FfmpegLibrary.isAvailable()) { - throw new FfmpegAudioDecoderException("Failed to load decoder native libraries."); + throw new FfmpegDecoderException("Failed to load decoder native libraries."); } Assertions.checkNotNull(format.sampleMimeType); - codecName = - Assertions.checkNotNull(FfmpegLibrary.getAudioCodecName(format.sampleMimeType)); + codecName = Assertions.checkNotNull(FfmpegLibrary.getCodecName(format.sampleMimeType)); extraData = getExtraData(format.sampleMimeType, format.initializationData); encoding = outputFloat ? C.ENCODING_PCM_FLOAT : C.ENCODING_PCM_16BIT; outputBufferSize = outputFloat ? OUTPUT_BUFFER_SIZE_32BIT : OUTPUT_BUFFER_SIZE_16BIT; nativeContext = ffmpegInitialize(codecName, extraData, outputFloat, format.sampleRate, format.channelCount); if (nativeContext == 0) { - throw new FfmpegAudioDecoderException("Initialization failed."); + throw new FfmpegDecoderException("Initialization failed."); } setInitialInputBufferSize(initialInputBufferSize); } @@ -89,21 +88,21 @@ protected DecoderInputBuffer createInputBuffer() { @Override protected SimpleOutputBuffer createOutputBuffer() { - return new SimpleOutputBuffer(this); + return new SimpleOutputBuffer(this::releaseOutputBuffer); } @Override - protected FfmpegAudioDecoderException createUnexpectedDecodeException(Throwable error) { - return new FfmpegAudioDecoderException("Unexpected decode error", error); + protected FfmpegDecoderException createUnexpectedDecodeException(Throwable error) { + return new FfmpegDecoderException("Unexpected decode error", error); } @Override - protected @Nullable FfmpegAudioDecoderException decode( - DecoderInputBuffer inputBuffer, SimpleOutputBuffer outputBuffer, boolean reset) { + protected @Nullable FfmpegDecoderException decode( + DecoderInputBuffer inputBuffer, SimpleOutputBuffer outputBuffer, boolean reset) { if (reset) { nativeContext = ffmpegReset(nativeContext, extraData); if (nativeContext == 0) { - return new FfmpegAudioDecoderException("Error resetting (see logcat)."); + return new FfmpegDecoderException("Error resetting (see logcat)."); } } ByteBuffer inputData = Util.castNonNull(inputBuffer.data); @@ -117,7 +116,7 @@ protected FfmpegAudioDecoderException createUnexpectedDecodeException(Throwable outputBuffer.setFlags(C.BUFFER_FLAG_DECODE_ONLY); return null; } else if (result == DECODER_ERROR_OTHER) { - return new FfmpegAudioDecoderException("Error decoding (see logcat)."); + return new FfmpegDecoderException("Error decoding (see logcat)."); } if (!hasOutputFormat) { channelCount = ffmpegGetChannelCount(nativeContext); diff --git a/extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegAudioRenderer.java b/extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegAudioRenderer.java index b67ba7b7307..168cbf3953d 100644 --- a/extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegAudioRenderer.java +++ b/extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegAudioRenderer.java @@ -39,7 +39,7 @@ public final class FfmpegAudioRenderer extends DecoderAudioRenderer { private final boolean enableFloatOutput; - private @MonotonicNonNull FfmpegDecoder decoder; + private @MonotonicNonNull FfmpegAudioDecoder decoder; public FfmpegAudioRenderer() { this(/* eventHandler= */ null, /* eventListener= */ null); @@ -110,12 +110,12 @@ public final int supportsMixedMimeTypeAdaptation() { } @Override - protected FfmpegDecoder createDecoder(Format format, @Nullable ExoMediaCrypto mediaCrypto) + protected FfmpegAudioDecoder createDecoder(Format format, @Nullable ExoMediaCrypto mediaCrypto) throws FfmpegDecoderException { int initialInputBufferSize = format.maxInputSize != Format.NO_VALUE ? format.maxInputSize : DEFAULT_INPUT_BUFFER_SIZE; decoder = - new FfmpegDecoder( + new FfmpegAudioDecoder( NUM_BUFFERS, NUM_BUFFERS, initialInputBufferSize, format, shouldUseFloatOutput(format)); return decoder; } diff --git a/extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegDecoder.java b/extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegDecoder.java deleted file mode 100644 index ceef86d0cc7..00000000000 --- a/extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegDecoder.java +++ /dev/null @@ -1,229 +0,0 @@ -/* - * Copyright (C) 2016 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.android.exoplayer2.ext.ffmpeg; - -import androidx.annotation.Nullable; -import com.google.android.exoplayer2.C; -import com.google.android.exoplayer2.Format; -import com.google.android.exoplayer2.decoder.DecoderInputBuffer; -import com.google.android.exoplayer2.decoder.SimpleDecoder; -import com.google.android.exoplayer2.decoder.SimpleOutputBuffer; -import com.google.android.exoplayer2.util.Assertions; -import com.google.android.exoplayer2.util.MimeTypes; -import com.google.android.exoplayer2.util.ParsableByteArray; -import com.google.android.exoplayer2.util.Util; -import java.nio.ByteBuffer; -import java.util.List; - -/** - * FFmpeg audio decoder. - */ -/* package */ final class FfmpegDecoder extends - SimpleDecoder { - - // Output buffer sizes when decoding PCM mu-law streams, which is the maximum FFmpeg outputs. - private static final int OUTPUT_BUFFER_SIZE_16BIT = 65536; - private static final int OUTPUT_BUFFER_SIZE_32BIT = OUTPUT_BUFFER_SIZE_16BIT * 2; - - // Error codes matching ffmpeg_jni.cc. - private static final int DECODER_ERROR_INVALID_DATA = -1; - private static final int DECODER_ERROR_OTHER = -2; - - private final String codecName; - @Nullable private final byte[] extraData; - private final @C.Encoding int encoding; - private final int outputBufferSize; - - private long nativeContext; // May be reassigned on resetting the codec. - private boolean hasOutputFormat; - private volatile int channelCount; - private volatile int sampleRate; - - public FfmpegDecoder( - int numInputBuffers, - int numOutputBuffers, - int initialInputBufferSize, - Format format, - boolean outputFloat) - throws FfmpegDecoderException { - super(new DecoderInputBuffer[numInputBuffers], new SimpleOutputBuffer[numOutputBuffers]); - if (!FfmpegLibrary.isAvailable()) { - throw new FfmpegDecoderException("Failed to load decoder native libraries."); - } - Assertions.checkNotNull(format.sampleMimeType); - codecName = Assertions.checkNotNull(FfmpegLibrary.getCodecName(format.sampleMimeType)); - extraData = getExtraData(format.sampleMimeType, format.initializationData); - encoding = outputFloat ? C.ENCODING_PCM_FLOAT : C.ENCODING_PCM_16BIT; - outputBufferSize = outputFloat ? OUTPUT_BUFFER_SIZE_32BIT : OUTPUT_BUFFER_SIZE_16BIT; - nativeContext = - ffmpegInitialize(codecName, extraData, outputFloat, format.sampleRate, format.channelCount); - if (nativeContext == 0) { - throw new FfmpegDecoderException("Initialization failed."); - } - setInitialInputBufferSize(initialInputBufferSize); - } - - @Override - public String getName() { - return "ffmpeg" + FfmpegLibrary.getVersion() + "-" + codecName; - } - - @Override - protected DecoderInputBuffer createInputBuffer() { - return new DecoderInputBuffer(DecoderInputBuffer.BUFFER_REPLACEMENT_MODE_DIRECT); - } - - @Override - protected SimpleOutputBuffer createOutputBuffer() { - return new SimpleOutputBuffer(this::releaseOutputBuffer); - } - - @Override - protected FfmpegDecoderException createUnexpectedDecodeException(Throwable error) { - return new FfmpegDecoderException("Unexpected decode error", error); - } - - @Override - protected @Nullable FfmpegDecoderException decode( - DecoderInputBuffer inputBuffer, SimpleOutputBuffer outputBuffer, boolean reset) { - if (reset) { - nativeContext = ffmpegReset(nativeContext, extraData); - if (nativeContext == 0) { - return new FfmpegDecoderException("Error resetting (see logcat)."); - } - } - ByteBuffer inputData = Util.castNonNull(inputBuffer.data); - int inputSize = inputData.limit(); - ByteBuffer outputData = outputBuffer.init(inputBuffer.timeUs, outputBufferSize); - int result = ffmpegDecode(nativeContext, inputData, inputSize, outputData, outputBufferSize); - if (result == DECODER_ERROR_INVALID_DATA) { - // Treat invalid data errors as non-fatal to match the behavior of MediaCodec. No output will - // be produced for this buffer, so mark it as decode-only to ensure that the audio sink's - // position is reset when more audio is produced. - outputBuffer.setFlags(C.BUFFER_FLAG_DECODE_ONLY); - return null; - } else if (result == DECODER_ERROR_OTHER) { - return new FfmpegDecoderException("Error decoding (see logcat)."); - } - if (!hasOutputFormat) { - channelCount = ffmpegGetChannelCount(nativeContext); - sampleRate = ffmpegGetSampleRate(nativeContext); - if (sampleRate == 0 && "alac".equals(codecName)) { - Assertions.checkNotNull(extraData); - // ALAC decoder did not set the sample rate in earlier versions of FFMPEG. - // See https://trac.ffmpeg.org/ticket/6096 - ParsableByteArray parsableExtraData = new ParsableByteArray(extraData); - parsableExtraData.setPosition(extraData.length - 4); - sampleRate = parsableExtraData.readUnsignedIntToInt(); - } - hasOutputFormat = true; - } - outputData.position(0); - outputData.limit(result); - return null; - } - - @Override - public void release() { - super.release(); - ffmpegRelease(nativeContext); - nativeContext = 0; - } - - /** Returns the channel count of output audio. */ - public int getChannelCount() { - return channelCount; - } - - /** Returns the sample rate of output audio. */ - public int getSampleRate() { - return sampleRate; - } - - /** - * Returns the encoding of output audio. - */ - public @C.Encoding int getEncoding() { - return encoding; - } - - /** - * Returns FFmpeg-compatible codec-specific initialization data ("extra data"), or {@code null} if - * not required. - */ - private static @Nullable byte[] getExtraData(String mimeType, List initializationData) { - switch (mimeType) { - case MimeTypes.AUDIO_AAC: - case MimeTypes.AUDIO_OPUS: - return initializationData.get(0); - case MimeTypes.AUDIO_ALAC: - return getAlacExtraData(initializationData); - case MimeTypes.AUDIO_VORBIS: - return getVorbisExtraData(initializationData); - default: - // Other codecs do not require extra data. - return null; - } - } - - private static byte[] getAlacExtraData(List initializationData) { - // FFmpeg's ALAC decoder expects an ALAC atom, which contains the ALAC "magic cookie", as extra - // data. initializationData[0] contains only the magic cookie, and so we need to package it into - // an ALAC atom. See: - // https://ffmpeg.org/doxygen/0.6/alac_8c.html - // https://github.com/macosforge/alac/blob/master/ALACMagicCookieDescription.txt - byte[] magicCookie = initializationData.get(0); - int alacAtomLength = 12 + magicCookie.length; - ByteBuffer alacAtom = ByteBuffer.allocate(alacAtomLength); - alacAtom.putInt(alacAtomLength); - alacAtom.putInt(0x616c6163); // type=alac - alacAtom.putInt(0); // version=0, flags=0 - alacAtom.put(magicCookie, /* offset= */ 0, magicCookie.length); - return alacAtom.array(); - } - - private static byte[] getVorbisExtraData(List initializationData) { - byte[] header0 = initializationData.get(0); - byte[] header1 = initializationData.get(1); - byte[] extraData = new byte[header0.length + header1.length + 6]; - extraData[0] = (byte) (header0.length >> 8); - extraData[1] = (byte) (header0.length & 0xFF); - System.arraycopy(header0, 0, extraData, 2, header0.length); - extraData[header0.length + 2] = 0; - extraData[header0.length + 3] = 0; - extraData[header0.length + 4] = (byte) (header1.length >> 8); - extraData[header0.length + 5] = (byte) (header1.length & 0xFF); - System.arraycopy(header1, 0, extraData, header0.length + 6, header1.length); - return extraData; - } - - private native long ffmpegInitialize( - String codecName, - @Nullable byte[] extraData, - boolean outputFloat, - int rawSampleRate, - int rawChannelCount); - - private native int ffmpegDecode(long context, ByteBuffer inputData, int inputSize, - ByteBuffer outputData, int outputSize); - private native int ffmpegGetChannelCount(long context); - private native int ffmpegGetSampleRate(long context); - - private native long ffmpegReset(long context, @Nullable byte[] extraData); - - private native void ffmpegRelease(long context); - -} diff --git a/extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegLibrary.java b/extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegLibrary.java index e3752aad5c9..c25bdec8132 100644 --- a/extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegLibrary.java +++ b/extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegLibrary.java @@ -32,8 +32,10 @@ public final class FfmpegLibrary { private static final String TAG = "FfmpegLibrary"; + // TODO: Use appropriate compilation script. private static final LibraryLoader LOADER = - new LibraryLoader("avutil", "swresample", "avcodec", "ffmpeg"); + new LibraryLoader("ffmpeg", "ffmpegJNI"); +// new LibraryLoader("avutil", "swresample", "avcodec", "ffmpeg"); private FfmpegLibrary() {} diff --git a/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegVideoDecoder.java b/extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegVideoDecoder.java similarity index 82% rename from extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegVideoDecoder.java rename to extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegVideoDecoder.java index 874299453f3..0e71f0fe051 100644 --- a/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegVideoDecoder.java +++ b/extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegVideoDecoder.java @@ -34,9 +34,9 @@ */ /* package */ final class FfmpegVideoDecoder extends - SimpleDecoder { + SimpleDecoder { - // Error codes matching ffmpeg_video_jni.cc. + // Error codes matching ffmpeg_jni.cc. private static final int DECODER_ERROR_INVALID_DATA = -1; private static final int DECODER_ERROR_OTHER = -2; private static final int DECODER_ERROR_READ_FRAME = -3; @@ -56,24 +56,24 @@ * @param numOutputBuffers Number of output buffers. * @param initialInputBufferSize The initial size of each input buffer, in bytes. * @param threads Number of threads libgav1 will use to decode. - * @throws FfmpegVideoDecoderException Thrown if an exception occurs when initializing the + * @throws FfmpegDecoderException Thrown if an exception occurs when initializing the * decoder. */ public FfmpegVideoDecoder( int numInputBuffers, int numOutputBuffers, int initialInputBufferSize, int threads, Format format) - throws FfmpegVideoDecoderException { + throws FfmpegDecoderException { super( new VideoDecoderInputBuffer[numInputBuffers], new VideoDecoderOutputBuffer[numOutputBuffers]); if (!FfmpegLibrary.isAvailable()) { - throw new FfmpegVideoDecoderException("Failed to load decoder native library."); + throw new FfmpegDecoderException("Failed to load decoder native library."); } - codecName = Assertions.checkNotNull(FfmpegLibrary.getVideoCodecName(format.sampleMimeType)); + codecName = Assertions.checkNotNull(FfmpegLibrary.getCodecName(format.sampleMimeType)); extraData = getExtraData(format.sampleMimeType, format.initializationData); this.format = format; nativeContext = ffmpegInitialize(codecName, extraData, threads); if (nativeContext == 0) { - throw new FfmpegVideoDecoderException("Failed to initialize decoder."); + throw new FfmpegDecoderException("Failed to initialize decoder."); } setInitialInputBufferSize(initialInputBufferSize); } @@ -126,12 +126,12 @@ protected VideoDecoderOutputBuffer createOutputBuffer() { @Override @Nullable - protected FfmpegVideoDecoderException decode( + protected FfmpegDecoderException decode( VideoDecoderInputBuffer inputBuffer, VideoDecoderOutputBuffer outputBuffer, boolean reset) { if (reset) { nativeContext = ffmpegReset(nativeContext); if (nativeContext == 0) { - return new FfmpegVideoDecoderException("Error resetting (see logcat)."); + return new FfmpegDecoderException("Error resetting (see logcat)."); } } @@ -149,7 +149,7 @@ protected FfmpegVideoDecoderException decode( // need read frame needSendAgain = true; } else if (sendPacketResult == DECODER_ERROR_OTHER) { - return new FfmpegVideoDecoderException("ffmpegDecode error: (see logcat)"); + return new FfmpegDecoderException("ffmpegDecode error: (see logcat)"); } // receive frame @@ -160,7 +160,7 @@ protected FfmpegVideoDecoderException decode( if (getFrameResult == DECODER_ERROR_SEND_PACKET) { return null; } else if (getFrameResult == DECODER_ERROR_OTHER) { - return new FfmpegVideoDecoderException("ffmpegDecode error: (see logcat)"); + return new FfmpegDecoderException("ffmpegDecode error: (see logcat)"); } if (getFrameResult == DECODER_ERROR_INVALID_DATA) { @@ -179,8 +179,8 @@ protected FfmpegVideoDecoderException decode( } @Override - protected FfmpegVideoDecoderException createUnexpectedDecodeException(Throwable error) { - return new FfmpegVideoDecoderException("Unexpected decode error", error); + protected FfmpegDecoderException createUnexpectedDecodeException(Throwable error) { + return new FfmpegDecoderException("Unexpected decode error", error); } @Override @@ -190,34 +190,24 @@ public void release() { nativeContext = 0; } - @Override - protected void releaseOutputBuffer(VideoDecoderOutputBuffer buffer) { - // Decode only frames do not acquire a reference on the internal decoder buffer and thus do not - // require a call to vpxReleaseFrame. -// if (outputMode == C.VIDEO_OUTPUT_MODE_SURFACE_YUV && !buffer.isDecodeOnly()) { -// gav1ReleaseFrame(nativeContext, buffer); -// } - super.releaseOutputBuffer(buffer); - } - /** * Renders output buffer to the given surface. Must only be called when in {@link * C#VIDEO_OUTPUT_MODE_SURFACE_YUV} mode. * * @param outputBuffer Output buffer. * @param surface Output surface. - * @throws FfmpegVideoDecoderException Thrown if called with invalid output mode or frame + * @throws FfmpegDecoderException Thrown if called with invalid output mode or frame * rendering fails. */ public void renderToSurface(VideoDecoderOutputBuffer outputBuffer, Surface surface) - throws FfmpegVideoDecoderException { + throws FfmpegDecoderException { if (outputBuffer.mode != C.VIDEO_OUTPUT_MODE_SURFACE_YUV) { - throw new FfmpegVideoDecoderException("Invalid output mode."); + throw new FfmpegDecoderException("Invalid output mode."); } if (ffmpegRenderFrame( nativeContext, surface, outputBuffer, outputBuffer.width, outputBuffer.height) == DECODER_ERROR_OTHER) { - throw new FfmpegVideoDecoderException( + throw new FfmpegDecoderException( "Buffer render error: "); } } diff --git a/extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegVideoRenderer.java b/extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegVideoRenderer.java index 8be8e7876ae..ed169ecc751 100644 --- a/extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegVideoRenderer.java +++ b/extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegVideoRenderer.java @@ -15,6 +15,8 @@ */ package com.google.android.exoplayer2.ext.ffmpeg; +import static java.lang.Runtime.getRuntime; + import android.os.Handler; import android.view.Surface; import androidx.annotation.Nullable; @@ -23,6 +25,10 @@ import com.google.android.exoplayer2.RendererCapabilities; import com.google.android.exoplayer2.decoder.Decoder; import com.google.android.exoplayer2.drm.ExoMediaCrypto; +import com.google.android.exoplayer2.util.Assertions; +import com.google.android.exoplayer2.util.MimeTypes; +import com.google.android.exoplayer2.util.TraceUtil; +import com.google.android.exoplayer2.util.Util; import com.google.android.exoplayer2.video.DecoderVideoRenderer; import com.google.android.exoplayer2.video.VideoDecoderInputBuffer; import com.google.android.exoplayer2.video.VideoDecoderOutputBuffer; @@ -36,6 +42,24 @@ */ public final class FfmpegVideoRenderer extends DecoderVideoRenderer { + private static final int DEFAULT_NUM_OF_INPUT_BUFFERS = 4; + private static final int DEFAULT_NUM_OF_OUTPUT_BUFFERS = 4; + /* Default size based on 720p resolution video compressed by a factor of two. */ + private static final int DEFAULT_INPUT_BUFFER_SIZE = + Util.ceilDivide(1280, 64) * Util.ceilDivide(720, 64) * (64 * 64 * 3 / 2) / 2; + + /** The number of input buffers. */ + private final int numInputBuffers; + /** + * The number of output buffers. The renderer may limit the minimum possible value due to + * requiring multiple output buffers to be dequeued at a time for it to make progress. + */ + private final int numOutputBuffers; + + private final int threads; + + @Nullable private FfmpegVideoDecoder decoder; + /** * Creates a new instance. * @@ -52,16 +76,47 @@ public FfmpegVideoRenderer( @Nullable Handler eventHandler, @Nullable VideoRendererEventListener eventListener, int maxDroppedFramesToNotify) { + this( + allowedJoiningTimeMs, + eventHandler, + eventListener, + maxDroppedFramesToNotify, + /* threads= */ getRuntime().availableProcessors(), + DEFAULT_NUM_OF_INPUT_BUFFERS, + DEFAULT_NUM_OF_OUTPUT_BUFFERS); + } + + /** + * Creates a new instance. + * + * @param allowedJoiningTimeMs The maximum duration in milliseconds for which this video renderer + * can attempt to seamlessly join an ongoing playback. + * @param eventHandler A handler to use when delivering events to {@code eventListener}. May be + * null if delivery of events is not required. + * @param eventListener A listener of events. May be null if delivery of events is not required. + * @param maxDroppedFramesToNotify The maximum number of frames that can be dropped between + * invocations of {@link VideoRendererEventListener#onDroppedFrames(int, long)}. + * @param threads Number of threads libgav1 will use to decode. + * @param numInputBuffers Number of input buffers. + * @param numOutputBuffers Number of output buffers. + */ + public FfmpegVideoRenderer( + long allowedJoiningTimeMs, + @Nullable Handler eventHandler, + @Nullable VideoRendererEventListener eventListener, + int maxDroppedFramesToNotify, + int threads, + int numInputBuffers, + int numOutputBuffers) { super(allowedJoiningTimeMs, eventHandler, eventListener, maxDroppedFramesToNotify); - // TODO: Implement. + this.threads = threads; + this.numInputBuffers = numInputBuffers; + this.numOutputBuffers = numOutputBuffers; } @Override @RendererCapabilities.Capabilities public final int supportsFormat(Format format) { - // TODO: Remove this line and uncomment the implementation below. - return FORMAT_UNSUPPORTED_TYPE; - /* String mimeType = Assertions.checkNotNull(format.sampleMimeType); if (!FfmpegLibrary.isAvailable() || !MimeTypes.isVideo(mimeType)) { return FORMAT_UNSUPPORTED_TYPE; @@ -75,32 +130,37 @@ public final int supportsFormat(Format format) { ADAPTIVE_SEAMLESS, TUNNELING_NOT_SUPPORTED); } - */ } - @SuppressWarnings("return.type.incompatible") @Override protected Decoder createDecoder(Format format, @Nullable ExoMediaCrypto mediaCrypto) throws FfmpegDecoderException { - // TODO: Implement, remove the SuppressWarnings annotation, and update the return type to use - // the concrete type of the decoder (probably FfmepgVideoDecoder). - return null; + TraceUtil.beginSection("createGav1Decoder"); + int initialInputBufferSize = + format.maxInputSize != Format.NO_VALUE ? format.maxInputSize : DEFAULT_INPUT_BUFFER_SIZE; + FfmpegVideoDecoder decoder = + new FfmpegVideoDecoder(numInputBuffers, numOutputBuffers, initialInputBufferSize, threads, format); + this.decoder = decoder; + TraceUtil.endSection(); + return decoder; } @Override protected void renderOutputBufferToSurface(VideoDecoderOutputBuffer outputBuffer, Surface surface) throws FfmpegDecoderException { - // TODO: Implement. + if (decoder == null) { + throw new FfmpegDecoderException( + "Failed to render output buffer to surface: decoder is not initialized."); + } + decoder.renderToSurface(outputBuffer, surface); + outputBuffer.release(); } @Override protected void setDecoderOutputMode(@C.VideoOutputMode int outputMode) { - // TODO: Uncomment the implementation below. - /* if (decoder != null) { decoder.setOutputMode(outputMode); } - */ } } diff --git a/extensions/ffmpegvideo/src/main/jni/CMakeLists.txt b/extensions/ffmpeg/src/main/jni/CMakeLists.txt similarity index 58% rename from extensions/ffmpegvideo/src/main/jni/CMakeLists.txt rename to extensions/ffmpeg/src/main/jni/CMakeLists.txt index de6720083f2..0d645ec6221 100644 --- a/extensions/ffmpegvideo/src/main/jni/CMakeLists.txt +++ b/extensions/ffmpeg/src/main/jni/CMakeLists.txt @@ -14,52 +14,51 @@ if(${ANDROID_ABI} MATCHES "armeabi-v7a") add_compile_options("-fPIC") endif() -set(libgav1_jni_root "${CMAKE_CURRENT_SOURCE_DIR}") -set(libgav1_jni_build "${CMAKE_BINARY_DIR}") -set(libgav1_jni_output_directory - ${libgav1_jni_root}/../libs/${ANDROID_ABI}/) +set(libgffmpeg_jni_root "${CMAKE_CURRENT_SOURCE_DIR}") +set(libgffmpeg_jni_build "${CMAKE_BINARY_DIR}") +set(libgffmpeg_jni_output_directory + ${libgffmpeg_jni_root}/../libs/${ANDROID_ABI}/) -#set(libgav1_root "${libgav1_jni_root}/libgav1") -#set(libgav1_build "${libgav1_jni_build}/libgav1") +#set(libgffmpeg_root "${libgffmpeg_jni_root}/libgffmpeg") +#set(libgffmpeg_build "${libgffmpeg_jni_build}/libgffmpeg") -#set(cpu_features_root "${libgav1_jni_root}/cpu_features") -#set(cpu_features_build "${libgav1_jni_build}/cpu_features") +#set(cpu_features_root "${libgffmpeg_jni_root}/cpu_features") +#set(cpu_features_build "${libgffmpeg_jni_build}/cpu_features") # Build cpu_features library. #add_subdirectory("${cpu_features_root}" # "${cpu_features_build}" # EXCLUDE_FROM_ALL) -# Build libgav1. -#add_subdirectory("${libgav1_root}" -# "${libgav1_build}" +# Build libgffmpeg. +#add_subdirectory("${libgffmpeg_root}" +# "${libgffmpeg_build}" # EXCLUDE_FROM_ALL) add_library(ffmpeg SHARED IMPORTED) set_target_properties(ffmpeg PROPERTIES IMPORTED_LOCATION - ${libgav1_jni_output_directory}/libffmpeg.so) + ${libgffmpeg_jni_output_directory}/libffmpeg.so) -# Build libgav1JNI. +# Build libgffmpegJNI. add_library(ffmpegJNI SHARED - ffmpeg_audio_jni.cc - ffmpeg_video_jni.cc) + ffmpeg_jni.cc) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include) # Locate NDK log library. find_library(android_log_lib log) -# Link libgav1JNI against used libraries. +# Link libgffmpegJNI against used libraries. target_link_libraries(ffmpegJNI PRIVATE android PRIVATE ffmpeg # PRIVATE cpu_features -# PRIVATE libgav1_static +# PRIVATE libgffmpeg_static PRIVATE ${android_log_lib}) -# Specify output directory for libgav1JNI. +# Specify output directory for libgffmpegJNI. set_target_properties(ffmpegJNI PROPERTIES LIBRARY_OUTPUT_DIRECTORY - ${libgav1_jni_output_directory}) + ${libgffmpeg_jni_output_directory}) diff --git a/extensions/ffmpeg/src/main/jni/ffmpeg_jni.cc b/extensions/ffmpeg/src/main/jni/ffmpeg_jni.cc index 400039af893..9365110da60 100644 --- a/extensions/ffmpeg/src/main/jni/ffmpeg_jni.cc +++ b/extensions/ffmpeg/src/main/jni/ffmpeg_jni.cc @@ -16,6 +16,10 @@ #include #include #include +#include +#include +#include +#include extern "C" { #ifdef __cplusplus @@ -36,14 +40,24 @@ extern "C" { #define LOGE(...) ((void)__android_log_print(ANDROID_LOG_ERROR, LOG_TAG, \ __VA_ARGS__)) -#define DECODER_FUNC(RETURN_TYPE, NAME, ...) \ +#define AUDIO_DECODER_FUNC(RETURN_TYPE, NAME, ...) \ extern "C" { \ JNIEXPORT RETURN_TYPE \ - Java_com_google_android_exoplayer2_ext_ffmpeg_FfmpegDecoder_ ## NAME \ + Java_com_google_android_exoplayer2_ext_ffmpeg_FfmpegAudioDecoder_ ## NAME \ (JNIEnv* env, jobject thiz, ##__VA_ARGS__);\ } \ JNIEXPORT RETURN_TYPE \ - Java_com_google_android_exoplayer2_ext_ffmpeg_FfmpegDecoder_ ## NAME \ + Java_com_google_android_exoplayer2_ext_ffmpeg_FfmpegAudioDecoder_ ## NAME \ + (JNIEnv* env, jobject thiz, ##__VA_ARGS__)\ + +#define VIDEO_DECODER_FUNC(RETURN_TYPE, NAME, ...) \ + extern "C" { \ + JNIEXPORT RETURN_TYPE \ + Java_com_google_android_exoplayer2_ext_ffmpeg_FfmpegVideoDecoder_ ## NAME \ + (JNIEnv* env, jobject thiz, ##__VA_ARGS__);\ + } \ + JNIEXPORT RETURN_TYPE \ + Java_com_google_android_exoplayer2_ext_ffmpeg_FfmpegVideoDecoder_ ## NAME \ (JNIEnv* env, jobject thiz, ##__VA_ARGS__)\ #define LIBRARY_FUNC(RETURN_TYPE, NAME, ...) \ @@ -63,10 +77,14 @@ static const AVSampleFormat OUTPUT_FORMAT_PCM_16BIT = AV_SAMPLE_FMT_S16; // Output format corresponding to AudioFormat.ENCODING_PCM_FLOAT. static const AVSampleFormat OUTPUT_FORMAT_PCM_FLOAT = AV_SAMPLE_FMT_FLT; -// Error codes matching FfmpegDecoder.java. +// Error codes matching FfmpegAudioDecoder.java. static const int DECODER_ERROR_INVALID_DATA = -1; static const int DECODER_ERROR_OTHER = -2; +// Error codes matching FfmpegVideoDecoder.java. +static const int DECODER_SUCCESS = 0; +static const int DECODER_ERROR_READ_FRAME = -3; + /** * Returns the AVCodec with the specified name, or NULL if it is not available. */ @@ -115,7 +133,7 @@ LIBRARY_FUNC(jboolean, ffmpegHasDecoder, jstring codecName) { return getCodecByName(env, codecName) != NULL; } -DECODER_FUNC(jlong, ffmpegInitialize, jstring codecName, jbyteArray extraData, +AUDIO_DECODER_FUNC(jlong, ffmpegInitialize, jstring codecName, jbyteArray extraData, jboolean outputFloat, jint rawSampleRate, jint rawChannelCount) { AVCodec *codec = getCodecByName(env, codecName); if (!codec) { @@ -126,7 +144,7 @@ DECODER_FUNC(jlong, ffmpegInitialize, jstring codecName, jbyteArray extraData, rawChannelCount); } -DECODER_FUNC(jint, ffmpegDecode, jlong context, jobject inputData, +AUDIO_DECODER_FUNC(jint, ffmpegDecode, jlong context, jobject inputData, jint inputSize, jobject outputData, jint outputSize) { if (!context) { LOGE("Context must be non-NULL."); @@ -154,7 +172,7 @@ DECODER_FUNC(jint, ffmpegDecode, jlong context, jobject inputData, outputSize); } -DECODER_FUNC(jint, ffmpegGetChannelCount, jlong context) { +AUDIO_DECODER_FUNC(jint, ffmpegGetChannelCount, jlong context) { if (!context) { LOGE("Context must be non-NULL."); return -1; @@ -162,7 +180,7 @@ DECODER_FUNC(jint, ffmpegGetChannelCount, jlong context) { return ((AVCodecContext *) context)->channels; } -DECODER_FUNC(jint, ffmpegGetSampleRate, jlong context) { +AUDIO_DECODER_FUNC(jint, ffmpegGetSampleRate, jlong context) { if (!context) { LOGE("Context must be non-NULL."); return -1; @@ -170,7 +188,7 @@ DECODER_FUNC(jint, ffmpegGetSampleRate, jlong context) { return ((AVCodecContext *) context)->sample_rate; } -DECODER_FUNC(jlong, ffmpegReset, jlong jContext, jbyteArray extraData) { +AUDIO_DECODER_FUNC(jlong, ffmpegReset, jlong jContext, jbyteArray extraData) { AVCodecContext *context = (AVCodecContext *) jContext; if (!context) { LOGE("Tried to reset without a context."); @@ -198,7 +216,7 @@ DECODER_FUNC(jlong, ffmpegReset, jlong jContext, jbyteArray extraData) { return (jlong) context; } -DECODER_FUNC(void, ffmpegRelease, jlong context) { +AUDIO_DECODER_FUNC(void, ffmpegRelease, jlong context) { if (context) { releaseContext((AVCodecContext *) context); } @@ -358,3 +376,347 @@ void releaseContext(AVCodecContext *context) { avcodec_free_context(&context); } + +// video + +// YUV plane indices. +const int kPlaneY = 0; +const int kPlaneU = 1; +const int kPlaneV = 2; +const int kMaxPlanes = 3; + +// Android YUV format. See: +// https://developer.android.com/reference/android/graphics/ImageFormat.html#YV12. +const int kImageFormatYV12 = 0x32315659; + +struct JniContext { + ~JniContext() { + if (native_window) { + ANativeWindow_release(native_window); + } + } + + bool MaybeAcquireNativeWindow(JNIEnv *env, jobject new_surface) { + if (surface == new_surface) { + return true; + } + if (native_window) { + ANativeWindow_release(native_window); + } + native_window_width = 0; + native_window_height = 0; + native_window = ANativeWindow_fromSurface(env, new_surface); + if (native_window == nullptr) { + LOGE("kJniStatusANativeWindowError"); + surface = nullptr; + return false; + } + surface = new_surface; + return true; + } + + jfieldID data_field; + jfieldID yuvPlanes_field; + jfieldID yuvStrides_field; + jmethodID init_for_private_frame_method; + jmethodID init_for_yuv_frame_method; + jmethodID init_method; + + AVCodecContext *codecContext; + + ANativeWindow *native_window = nullptr; + jobject surface = nullptr; + int native_window_width = 0; + int native_window_height = 0; +}; + +void CopyPlane(const uint8_t *source, int source_stride, uint8_t *destination, + int destination_stride, int width, int height) { + while (height--) { + std::memcpy(destination, source, width); + source += source_stride; + destination += destination_stride; + } +} + +constexpr int AlignTo16(int value) { return (value + 15) & (~15); } + +JniContext *createVideoContext(JNIEnv *env, + AVCodec *codec, + jbyteArray extraData, + jint threads); + +JniContext *createVideoContext(JNIEnv *env, + AVCodec *codec, + jbyteArray extraData, + jint threads) { + JniContext *jniContext = new(std::nothrow)JniContext(); + + AVCodecContext *codecContext = avcodec_alloc_context3(codec); + if (!codecContext) { + LOGE("Failed to allocate context."); + return NULL; + } + + if (extraData) { + jsize size = env->GetArrayLength(extraData); + codecContext->extradata_size = size; + codecContext->extradata = + (uint8_t *) av_malloc(size + AV_INPUT_BUFFER_PADDING_SIZE); + if (!codecContext->extradata) { + LOGE("Failed to allocate extradata."); + releaseContext(codecContext); + return NULL; + } + env->GetByteArrayRegion(extraData, 0, size, (jbyte *) codecContext->extradata); + } + + codecContext->thread_count = threads; + codecContext->err_recognition = AV_EF_IGNORE_ERR; + int result = avcodec_open2(codecContext, codec, NULL); + if (result < 0) { + logError("avcodec_open2", result); + releaseContext(codecContext); + return NULL; + } + + jniContext->codecContext = codecContext; + + // Populate JNI References. + const jclass outputBufferClass = env->FindClass( + "com/google/android/exoplayer2/video/VideoDecoderOutputBuffer"); + jniContext->data_field = env->GetFieldID(outputBufferClass, "data", "Ljava/nio/ByteBuffer;"); + jniContext->yuvPlanes_field = + env->GetFieldID(outputBufferClass, "yuvPlanes", "[Ljava/nio/ByteBuffer;"); + jniContext->yuvStrides_field = env->GetFieldID(outputBufferClass, "yuvStrides", "[I"); + jniContext->init_for_private_frame_method = + env->GetMethodID(outputBufferClass, "initForPrivateFrame", "(II)V"); + jniContext->init_for_yuv_frame_method = + env->GetMethodID(outputBufferClass, "initForYuvFrame", "(IIIII)Z"); + jniContext->init_method = + env->GetMethodID(outputBufferClass, "init", "(JILjava/nio/ByteBuffer;)V"); + + return jniContext; +} + +VIDEO_DECODER_FUNC(jlong, ffmpegInitialize, jstring codecName, jbyteArray extraData, jint threads) { + AVCodec *codec = getCodecByName(env, codecName); + if (!codec) { + LOGE("Codec not found."); + return 0L; + } + + return (jlong) createVideoContext(env, codec, extraData, threads); +} + +VIDEO_DECODER_FUNC(jlong, ffmpegReset, jlong jContext) { + JniContext *const jniContext = reinterpret_cast(jContext); + AVCodecContext *context = jniContext->codecContext; + if (!context) { + LOGE("Tried to reset without a context."); + return 0L; + } + + avcodec_flush_buffers(context); + return (jlong) jniContext; +} + +VIDEO_DECODER_FUNC(void, ffmpegRelease, jlong jContext) { + JniContext *const jniContext = reinterpret_cast(jContext); + AVCodecContext *context = jniContext->codecContext; + if (context) { + releaseContext(context); + } +} + + +VIDEO_DECODER_FUNC(jint, ffmpegSendPacket, jlong jContext, jobject encodedData, + jint length, jlong inputTimeUs) { + JniContext *const jniContext = reinterpret_cast(jContext); + AVCodecContext *avContext = jniContext->codecContext; + + uint8_t *inputBuffer = (uint8_t *) env->GetDirectBufferAddress(encodedData); + AVPacket packet; + av_init_packet(&packet); + packet.data = inputBuffer; + packet.size = length; + packet.pts = inputTimeUs; + + int result = 0; + // Queue input data. + result = avcodec_send_packet(avContext, &packet); + if (result) { + logError("avcodec_send_packet", result); + if (result == AVERROR_INVALIDDATA) { + // need more data + return DECODER_ERROR_INVALID_DATA; + } else if (result == AVERROR(EAGAIN)) { + // need read frame + return DECODER_ERROR_READ_FRAME; + } else { + return DECODER_ERROR_OTHER; + } + } + return result; +} + +VIDEO_DECODER_FUNC(jint, ffmpegReceiveFrame, jlong jContext, jint outputMode, jobject jOutputBuffer, + jboolean decodeOnly) { + JniContext *const jniContext = reinterpret_cast(jContext); + AVCodecContext *avContext = jniContext->codecContext; + int result = 0; + + AVFrame *frame = av_frame_alloc(); + if (!frame) { + LOGE("Failed to allocate output frame."); + return DECODER_ERROR_OTHER; + } + result = avcodec_receive_frame(avContext, frame); + + // fail + if (decodeOnly || result == AVERROR(EAGAIN)) { + // This is not an error. The input data was decode-only or no displayable + // frames are available. + av_frame_free(&frame); + return DECODER_ERROR_INVALID_DATA; + } + if (result) { + av_frame_free(&frame); + logError("avcodec_receive_frame", result); + return DECODER_ERROR_OTHER; + } + + // success + // init time and mode + env->CallVoidMethod(jOutputBuffer, jniContext->init_method, frame->pts, outputMode, nullptr); + + // init data + const jboolean init_result = env->CallBooleanMethod( + jOutputBuffer, jniContext->init_for_yuv_frame_method, + frame->width, + frame->height, + frame->linesize[0], frame->linesize[1], + 0); + if (env->ExceptionCheck()) { + // Exception is thrown in Java when returning from the native call. + return DECODER_ERROR_OTHER; + } + if (!init_result) { + return DECODER_ERROR_OTHER; + } + + const jobject data_object = env->GetObjectField(jOutputBuffer, jniContext->data_field); + jbyte *data = reinterpret_cast(env->GetDirectBufferAddress(data_object)); + const int32_t uvHeight = (frame->height + 1) / 2; + const uint64_t yLength = frame->linesize[0] * frame->height; + const uint64_t uvLength = frame->linesize[1] * uvHeight; + + // TODO: Support rotate YUV data + + memcpy(data, frame->data[0], yLength); + memcpy(data + yLength, frame->data[1], uvLength); + memcpy(data + yLength + uvLength, frame->data[2], uvLength); + + av_frame_free(&frame); + + return result; +} + +VIDEO_DECODER_FUNC(jint, ffmpegRenderFrame, jlong jContext, jobject jSurface, + jobject jOutputBuffer, jint displayedWidth, jint displayedHeight) { + JniContext *const jniContext = reinterpret_cast(jContext); + if (!jniContext->MaybeAcquireNativeWindow(env, jSurface)) { + return DECODER_ERROR_OTHER; + } + + if (jniContext->native_window_width != displayedWidth || + jniContext->native_window_height != displayedHeight) { + if (ANativeWindow_setBuffersGeometry( + jniContext->native_window, + displayedWidth, + displayedHeight, + kImageFormatYV12)) { + LOGE("kJniStatusANativeWindowError"); + return DECODER_ERROR_OTHER; + } + jniContext->native_window_width = displayedWidth; + jniContext->native_window_height = displayedHeight; + } + + ANativeWindow_Buffer native_window_buffer; + if (ANativeWindow_lock(jniContext->native_window, &native_window_buffer, +/*inOutDirtyBounds=*/nullptr) || + native_window_buffer.bits == nullptr) { + LOGE("kJniStatusANativeWindowError"); + return DECODER_ERROR_OTHER; + } + + jobject yuvPlanes_object = env->GetObjectField(jOutputBuffer, jniContext->yuvPlanes_field); + jobjectArray yuvPlanes_array = static_cast(yuvPlanes_object); + jobject yuvPlanesY = env->GetObjectArrayElement(yuvPlanes_array, kPlaneY); + jobject yuvPlanesU = env->GetObjectArrayElement(yuvPlanes_array, kPlaneU); + jobject yuvPlanesV = env->GetObjectArrayElement(yuvPlanes_array, kPlaneV); + jbyte *planeY = reinterpret_cast(env->GetDirectBufferAddress(yuvPlanesY)); + jbyte *planeU = reinterpret_cast(env->GetDirectBufferAddress(yuvPlanesU)); + jbyte *planeV = reinterpret_cast(env->GetDirectBufferAddress(yuvPlanesV)); + + jobject yuvStrides_object = env->GetObjectField(jOutputBuffer, jniContext->yuvStrides_field); + jintArray *yuvStrides_array = reinterpret_cast(&yuvStrides_object); + + int *yuvStrides = env->GetIntArrayElements(*yuvStrides_array, NULL); + int strideY = yuvStrides[kPlaneY]; + int strideU = yuvStrides[kPlaneU]; + int strideV = yuvStrides[kPlaneV]; + + // Y plane + CopyPlane(reinterpret_cast(planeY), + strideY, + reinterpret_cast(native_window_buffer.bits), + native_window_buffer.stride, + displayedWidth, + displayedHeight); + + const int y_plane_size = + native_window_buffer.stride * native_window_buffer.height; + const int32_t native_window_buffer_uv_height = + (native_window_buffer.height + 1) / 2; + const int native_window_buffer_uv_stride = + AlignTo16(native_window_buffer.stride / 2); + + // TODO(b/140606738): Handle monochrome videos. + + // V plane + // Since the format for ANativeWindow is YV12, V plane is being processed + // before U plane. + const int v_plane_height = std::min(native_window_buffer_uv_height, + displayedHeight); + CopyPlane( + reinterpret_cast(planeV), + strideV, + reinterpret_cast(native_window_buffer.bits) + y_plane_size, + native_window_buffer_uv_stride, displayedWidth, + v_plane_height); + + const int v_plane_size = v_plane_height * native_window_buffer_uv_stride; + + // U plane + CopyPlane( + reinterpret_cast(planeU), + strideU, + reinterpret_cast(native_window_buffer.bits) + + y_plane_size + v_plane_size, + native_window_buffer_uv_stride, displayedWidth, + std::min(native_window_buffer_uv_height, + displayedHeight)); + + + env->ReleaseIntArrayElements(*yuvStrides_array, yuvStrides, 0); + + if (ANativeWindow_unlockAndPost(jniContext->native_window)) { + LOGE("kJniStatusANativeWindowError"); + return DECODER_ERROR_OTHER; + } + + return DECODER_SUCCESS; +} + diff --git a/extensions/ffmpegvideo/README.md b/extensions/ffmpegvideo/README.md deleted file mode 100644 index 7cdab350ae9..00000000000 --- a/extensions/ffmpegvideo/README.md +++ /dev/null @@ -1,86 +0,0 @@ -# ExoPlayer FFmpeg extension # - -The Ffmpeg extension provides `FfmpegAudioRenderer` and `FfmpegVideoRenderer`, which uses FFmpeg -native library to decode videos. - -***This extension is currently in its very infancy and is under development.*** - -***Whats working?*** -video supported codec: only H.264 -audio supported codec: same as original extension -supported surface type: video_decoder_gl_surface_view - -***On Plan:*** -- [ ] Support other surface types -- [ ] Organize the code -- [ ] Fix possible issues -- [ ] Video Decoder support Format.rotationDegrees -- [ ] Support other codecs - - -## License note ## - -Please note that whilst the code in this repository is licensed under -[Apache 2.0][], using this extension also requires building and including one or -more external libraries as described below. These are licensed separately. - -[Apache 2.0]: https://github.com/google/ExoPlayer/blob/release-v2/LICENSE - -## Build instructions (Linux, macOS) ## - -To use this extension you need to clone the ExoPlayer repository and depend on -its modules locally. Instructions for doing this can be found in ExoPlayer's -[top level README][]. - -I provided the compiled FFmpeg [*.so files][] and [header files][]. You need to copy -the .so files to the `src/main/libs` directory and the header files to -the `src/main/jni/include` directory. Of course you can also compile it yourself. - - -## Using the extension ## - -Like av1 extension, pass `EXTENSION_RENDERER_MODE_PREFER`, use `FFmpegRenderersFactory` -instead of `DefaultRenderersFactory` to create `FfmpegVideoRenderer` and `FfmpegAudioRenderer`. -Then you can observe the related logs of `EventLogger#decoderInitialized` in logcat -to determine whether the ffmpeg extension is used correctly. - -## Using the extension in the demo application ## - -To try out playback using the extension in the [demo application][], see -[enabling extension decoders][]. - -use `FFmpegRenderersFactory` instead of `DefaultRenderersFactory`. - -[demo application]: https://exoplayer.dev/demo-application.html -[enabling extension decoders]: https://exoplayer.dev/demo-application.html#enabling-extension-decoders - -## Rendering options ## - -There are two possibilities for rendering the output `Libgav1VideoRenderer` -gets from the libgav1 decoder: - -* GL rendering using GL shader for color space conversion - * If you are using `SimpleExoPlayer` with `PlayerView`, enable this option by - setting `surface_type` of `PlayerView` to be - `video_decoder_gl_surface_view`. - * Otherwise, enable this option by sending `Libgav1VideoRenderer` a message - of type `C.MSG_SET_VIDEO_DECODER_OUTPUT_BUFFER_RENDERER` with an instance of - `VideoDecoderOutputBufferRenderer` as its object. - -* Native rendering using `ANativeWindow` - * If you are using `SimpleExoPlayer` with `PlayerView`, this option is enabled - by default. - * Otherwise, enable this option by sending `Libgav1VideoRenderer` a message of - type `C.MSG_SET_SURFACE` with an instance of `SurfaceView` as its object. - -Note: Although the default option uses `ANativeWindow`, based on our testing the -GL rendering mode has better performance, so should be preferred - -## Links ## - -* [Javadoc][]: Classes matching `com.google.android.exoplayer2.ext.av1.*` - belong to this module. - -[Javadoc]: https://exoplayer.dev/doc/reference/index.html -[*.so files]: https://drive.google.com/open?id=14v4tz5L_jU7di3xWrY-uhuS7K5mcwj3g -[header files]: https://drive.google.com/open?id=1dDZ9R4cLPpgcHOCoUpClrOlqnGL2UTSr diff --git a/extensions/ffmpegvideo/build.gradle b/extensions/ffmpegvideo/build.gradle deleted file mode 100644 index 40278a439ef..00000000000 --- a/extensions/ffmpegvideo/build.gradle +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright (C) 2019 The Android Open Source Project -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -apply from: '../../constants.gradle' -apply plugin: 'com.android.library' - -android { - compileSdkVersion project.ext.compileSdkVersion - - compileOptions { - sourceCompatibility JavaVersion.VERSION_1_8 - targetCompatibility JavaVersion.VERSION_1_8 - } - - defaultConfig { - minSdkVersion project.ext.minSdkVersion - targetSdkVersion project.ext.targetSdkVersion - consumerProguardFiles 'proguard-rules.txt' - - externalNativeBuild { - cmake { - // Debug CMake build type causes video frames to drop, - // so native library should always use Release build type. - arguments "-DCMAKE_BUILD_TYPE=Release" - targets "ffmpegJNI" - } - } - } - - externalNativeBuild { - cmake { - version '3.10.2' - path "src/main/jni/CMakeLists.txt" - } - } - - buildTypes { - debug { - ndk { - abiFilters 'arm64-v8a'/*, 'x86_64'*/ - } - } - } - - // This option resolves the problem of finding libgav1JNI.so - // on multiple paths. The first one found is picked. - packagingOptions { - pickFirst 'lib/arm64-v8a/libffmpegJNI.so' - pickFirst 'lib/armeabi-v7a/libffmpegJNI.so' - pickFirst 'lib/x86/libffmpegJNI.so' - pickFirst 'lib/x86_64/libffmpegJNI.so' - } - - sourceSets.main { - // As native JNI library build is invoked from gradle, this is - // not needed. However, it exposes the built library and keeps - // consistency with the other extensions. - jniLibs.srcDir 'src/main/libs' - } -} - -// Configure the native build only if libgav1 is present, to avoid gradle sync -// failures if libgav1 hasn't been checked out according to the README and CMake -// isn't installed. -//if (project.file('src/main/jni/libffmpeg').exists()) { -// android.externalNativeBuild.cmake.path = 'src/main/jni/CMakeLists.txt' -// android.externalNativeBuild.cmake.version = '3.7.1+' -//} - -dependencies { - implementation project(modulePrefix + 'library-core') - implementation 'androidx.annotation:annotation:' + androidxAnnotationVersion - compileOnly 'org.checkerframework:checker-qual:' + checkerframeworkVersion - compileOnly 'org.jetbrains.kotlin:kotlin-annotations-jvm:' + kotlinAnnotationsVersion -} - -ext { - javadocTitle = 'FFmpeg video extension' -} -apply from: '../../javadoc_library.gradle' diff --git a/extensions/ffmpegvideo/proguard-rules.txt b/extensions/ffmpegvideo/proguard-rules.txt deleted file mode 100644 index 9d73f7e2b58..00000000000 --- a/extensions/ffmpegvideo/proguard-rules.txt +++ /dev/null @@ -1,7 +0,0 @@ -# Proguard rules specific to the AV1 extension. - -# This prevents the names of native methods from being obfuscated. --keepclasseswithmembernames class * { - native ; -} - diff --git a/extensions/ffmpegvideo/src/main/AndroidManifest.xml b/extensions/ffmpegvideo/src/main/AndroidManifest.xml deleted file mode 100644 index d53bca4ca22..00000000000 --- a/extensions/ffmpegvideo/src/main/AndroidManifest.xml +++ /dev/null @@ -1,17 +0,0 @@ - - - - diff --git a/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FFmpegRenderersFactory.java b/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FFmpegRenderersFactory.java deleted file mode 100644 index 229589dff23..00000000000 --- a/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FFmpegRenderersFactory.java +++ /dev/null @@ -1,137 +0,0 @@ -package com.google.android.exoplayer2.ext.ffmpeg; - -import android.content.Context; -import android.os.Handler; -import com.google.android.exoplayer2.DefaultRenderersFactory; -import com.google.android.exoplayer2.Renderer; -import com.google.android.exoplayer2.audio.AudioCapabilities; -import com.google.android.exoplayer2.audio.AudioProcessor; -import com.google.android.exoplayer2.audio.AudioRendererEventListener; -import com.google.android.exoplayer2.audio.DefaultAudioSink; -import com.google.android.exoplayer2.audio.MediaCodecAudioRenderer; -import com.google.android.exoplayer2.mediacodec.MediaCodecSelector; -import com.google.android.exoplayer2.util.Log; -import com.google.android.exoplayer2.video.MediaCodecVideoRenderer; -import com.google.android.exoplayer2.video.VideoRendererEventListener; -import java.lang.reflect.Constructor; -import java.util.ArrayList; - -public class FFmpegRenderersFactory extends DefaultRenderersFactory { - - private static final String TAG = "FFmpegRenderersFactory"; - - public FFmpegRenderersFactory(Context context) { - super(context); - } - - @Override - protected void buildVideoRenderers( - Context context, - @ExtensionRendererMode int extensionRendererMode, - MediaCodecSelector mediaCodecSelector, - boolean enableDecoderFallback, - Handler eventHandler, - VideoRendererEventListener eventListener, - long allowedVideoJoiningTimeMs, - ArrayList out) { - MediaCodecVideoRenderer videoRenderer = - new MediaCodecVideoRenderer( - context, - mediaCodecSelector, - allowedVideoJoiningTimeMs, - enableDecoderFallback, - eventHandler, - eventListener, - MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY); - out.add(videoRenderer); - - if (extensionRendererMode == EXTENSION_RENDERER_MODE_OFF) { - return; - } - int extensionRendererIndex = out.size(); - if (extensionRendererMode == EXTENSION_RENDERER_MODE_PREFER) { - extensionRendererIndex--; - } - - try { - // Full class names used for constructor args so the LINT rule triggers if any of them move. - // LINT.IfChange - Class clazz = Class - .forName("com.google.android.exoplayer2.ext.ffmpeg.FfmpegVideoRenderer"); - Constructor constructor = - clazz.getConstructor( - long.class, - Handler.class, - com.google.android.exoplayer2.video.VideoRendererEventListener.class, - int.class); - // LINT.ThenChange(../../../../../../../proguard-rules.txt) - Renderer renderer = - (Renderer) - constructor.newInstance( - allowedVideoJoiningTimeMs, - eventHandler, - eventListener, - MAX_DROPPED_VIDEO_FRAME_COUNT_TO_NOTIFY); - out.add(extensionRendererIndex++, renderer); - Log.i(TAG, "Loaded FfmpegVideoRenderer."); - } catch (ClassNotFoundException e) { - // Expected if the app was built without the extension. - } catch (Exception e) { - // The extension is present, but instantiation failed. - throw new RuntimeException("Error instantiating Ffmpeg extension", e); - } - - } - - @Override - protected void buildAudioRenderers( - Context context, - int extensionRendererMode, - MediaCodecSelector mediaCodecSelector, - boolean enableDecoderFallback, - AudioProcessor[] audioProcessors, - Handler eventHandler, - AudioRendererEventListener eventListener, - ArrayList out) { - MediaCodecAudioRenderer audioRenderer = - new MediaCodecAudioRenderer( - context, - mediaCodecSelector, - enableDecoderFallback, - eventHandler, - eventListener, - new DefaultAudioSink(AudioCapabilities.getCapabilities(context), audioProcessors)); - out.add(audioRenderer); - - if (extensionRendererMode == EXTENSION_RENDERER_MODE_OFF) { - return; - } - int extensionRendererIndex = out.size(); - if (extensionRendererMode == EXTENSION_RENDERER_MODE_PREFER) { - extensionRendererIndex--; - } - - try { - // Full class names used for constructor args so the LINT rule triggers if any of them move. - // LINT.IfChange - Class clazz = - Class.forName("com.google.android.exoplayer2.ext.ffmpeg.FfmpegAudioRenderer"); - Constructor constructor = - clazz.getConstructor( - android.os.Handler.class, - com.google.android.exoplayer2.audio.AudioRendererEventListener.class, - com.google.android.exoplayer2.audio.AudioProcessor[].class); - // LINT.ThenChange(../../../../../../../proguard-rules.txt) - Renderer renderer = - (Renderer) constructor.newInstance(eventHandler, eventListener, audioProcessors); - out.add(extensionRendererIndex++, renderer); - Log.i(TAG, "Loaded FfmpegAudioRenderer."); - } catch (ClassNotFoundException e) { - // Expected if the app was built without the extension. - } catch (Exception e) { - // The extension is present, but instantiation failed. - throw new RuntimeException("Error instantiating FFmpeg extension", e); - } - } - -} diff --git a/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegAudioDecoderException.java b/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegAudioDecoderException.java deleted file mode 100644 index 82aac5a9ef4..00000000000 --- a/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegAudioDecoderException.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright (C) 2016 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.android.exoplayer2.ext.ffmpeg; - -import com.google.android.exoplayer2.audio.AudioDecoderException; - -/** - * Thrown when an FFmpeg decoder error occurs. - */ -public final class FfmpegAudioDecoderException extends AudioDecoderException { - - /* package */ FfmpegAudioDecoderException(String message) { - super(message); - } - - /* package */ FfmpegAudioDecoderException(String message, Throwable cause) { - super(message, cause); - } -} diff --git a/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegAudioRenderer.java b/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegAudioRenderer.java deleted file mode 100644 index 726a367160b..00000000000 --- a/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegAudioRenderer.java +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Copyright (C) 2016 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.android.exoplayer2.ext.ffmpeg; - -import android.os.Handler; -import androidx.annotation.Nullable; -import com.google.android.exoplayer2.C; -import com.google.android.exoplayer2.Format; -import com.google.android.exoplayer2.audio.AudioProcessor; -import com.google.android.exoplayer2.audio.AudioRendererEventListener; -import com.google.android.exoplayer2.audio.AudioSink; -import com.google.android.exoplayer2.audio.DefaultAudioSink; -import com.google.android.exoplayer2.audio.SimpleDecoderAudioRenderer; -import com.google.android.exoplayer2.drm.ExoMediaCrypto; -import com.google.android.exoplayer2.util.Assertions; -import com.google.android.exoplayer2.util.MimeTypes; -import org.checkerframework.checker.nullness.qual.MonotonicNonNull; - -/** - * Decodes and renders audio using FFmpeg. - */ -public final class FfmpegAudioRenderer extends SimpleDecoderAudioRenderer { - - /** The number of input and output buffers. */ - private static final int NUM_BUFFERS = 16; - /** The default input buffer size. */ - private static final int DEFAULT_INPUT_BUFFER_SIZE = 960 * 6; - - private final boolean enableFloatOutput; - - private @MonotonicNonNull FfmpegAudioDecoder decoder; - - public FfmpegAudioRenderer() { - this(/* eventHandler= */ null, /* eventListener= */ null); - } - - /** - * @param eventHandler A handler to use when delivering events to {@code eventListener}. May be - * null if delivery of events is not required. - * @param eventListener A listener of events. May be null if delivery of events is not required. - * @param audioProcessors Optional {@link AudioProcessor}s that will process audio before output. - */ - public FfmpegAudioRenderer( - @Nullable Handler eventHandler, - @Nullable AudioRendererEventListener eventListener, - AudioProcessor... audioProcessors) { - this( - eventHandler, - eventListener, - new DefaultAudioSink(/* audioCapabilities= */ null, audioProcessors), - /* enableFloatOutput= */ false); - } - - /** - * @param eventHandler A handler to use when delivering events to {@code eventListener}. May be - * null if delivery of events is not required. - * @param eventListener A listener of events. May be null if delivery of events is not required. - * @param audioSink The sink to which audio will be output. - * @param enableFloatOutput Whether to enable 32-bit float audio format, if supported on the - * device/build and if the input format may have bit depth higher than 16-bit. When using - * 32-bit float output, any audio processing will be disabled, including playback speed/pitch - * adjustment. - */ - public FfmpegAudioRenderer( - @Nullable Handler eventHandler, - @Nullable AudioRendererEventListener eventListener, - AudioSink audioSink, - boolean enableFloatOutput) { - super( - eventHandler, - eventListener, - audioSink); - this.enableFloatOutput = enableFloatOutput; - } - - @Override - @FormatSupport - protected int supportsFormatInternal(Format format) { - Assertions.checkNotNull(format.sampleMimeType); - if (!FfmpegLibrary.isAvailable()) { - return FORMAT_UNSUPPORTED_TYPE; - } else if (!FfmpegLibrary.supportsAudioFormat(format.sampleMimeType) || !isOutputSupported(format)) { - return FORMAT_UNSUPPORTED_SUBTYPE; - } else if (format.drmInitData != null && format.exoMediaCryptoType == null) { - return FORMAT_UNSUPPORTED_DRM; - } else { - return FORMAT_HANDLED; - } - } - - @Override - @AdaptiveSupport - public final int supportsMixedMimeTypeAdaptation() { - return ADAPTIVE_NOT_SEAMLESS; - } - - @Override - protected FfmpegAudioDecoder createDecoder(Format format, @Nullable ExoMediaCrypto mediaCrypto) - throws FfmpegAudioDecoderException { - int initialInputBufferSize = - format.maxInputSize != Format.NO_VALUE ? format.maxInputSize : DEFAULT_INPUT_BUFFER_SIZE; - decoder = - new FfmpegAudioDecoder( - NUM_BUFFERS, NUM_BUFFERS, initialInputBufferSize, format, shouldUseFloatOutput(format)); - return decoder; - } - - @Override - public Format getOutputFormat() { - Assertions.checkNotNull(decoder); - return new Format.Builder() - .setSampleMimeType(MimeTypes.AUDIO_RAW) - .setChannelCount(decoder.getChannelCount()) - .setSampleRate(decoder.getSampleRate()) - .setPcmEncoding(decoder.getEncoding()) - .build(); - } - - private boolean isOutputSupported(Format inputFormat) { - return shouldUseFloatOutput(inputFormat) - || supportsOutput(inputFormat.channelCount, C.ENCODING_PCM_16BIT); - } - - private boolean shouldUseFloatOutput(Format inputFormat) { - Assertions.checkNotNull(inputFormat.sampleMimeType); - if (!enableFloatOutput || !supportsOutput(inputFormat.channelCount, C.ENCODING_PCM_FLOAT)) { - return false; - } - switch (inputFormat.sampleMimeType) { - case MimeTypes.AUDIO_RAW: - // For raw audio, output in 32-bit float encoding if the bit depth is > 16-bit. - return inputFormat.pcmEncoding == C.ENCODING_PCM_24BIT - || inputFormat.pcmEncoding == C.ENCODING_PCM_32BIT - || inputFormat.pcmEncoding == C.ENCODING_PCM_FLOAT; - case MimeTypes.AUDIO_AC3: - // AC-3 is always 16-bit, so there is no point outputting in 32-bit float encoding. - return false; - default: - // For all other formats, assume that it's worth using 32-bit float encoding. - return true; - } - } - -} diff --git a/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegLibrary.java b/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegLibrary.java deleted file mode 100644 index 6bcbf3693f7..00000000000 --- a/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegLibrary.java +++ /dev/null @@ -1,164 +0,0 @@ -/* - * Copyright (C) 2016 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.android.exoplayer2.ext.ffmpeg; - -import androidx.annotation.Nullable; -import com.google.android.exoplayer2.ExoPlayerLibraryInfo; -import com.google.android.exoplayer2.util.LibraryLoader; -import com.google.android.exoplayer2.util.Log; -import com.google.android.exoplayer2.util.MimeTypes; - -/** - * Configures and queries the underlying native library. - */ -public final class FfmpegLibrary { - - static { - ExoPlayerLibraryInfo.registerModule("goog.exo.ffmpegvideo"); - } - - private static final String TAG = "FfmpegLibrary"; - - private static final LibraryLoader LOADER = - new LibraryLoader("ffmpeg", "ffmpegJNI"); - - private FfmpegLibrary() {} - - /** - * Override the names of the FFmpeg native libraries. If an application wishes to call this - * method, it must do so before calling any other method defined by this class, and before - * instantiating a {@link FfmpegAudioRenderer} instance. - * - * @param libraries The names of the FFmpeg native libraries. - */ - public static void setLibraries(String... libraries) { - LOADER.setLibraries(libraries); - } - - /** - * Returns whether the underlying library is available, loading it if necessary. - */ - public static boolean isAvailable() { - return LOADER.isAvailable(); - } - - /** Returns the version of the underlying library if available, or null otherwise. */ - public static @Nullable String getVersion() { - return isAvailable() ? ffmpegGetVersion() : null; - } - - /** - * Returns whether the underlying library supports the specified MIME type. - * - * @param mimeType The MIME type to check. - */ - public static boolean supportsAudioFormat(String mimeType) { - if (!isAvailable()) { - return false; - } - String codecName = getAudioCodecName(mimeType); - if (codecName == null) { - return false; - } - if (!ffmpegHasDecoder(codecName)) { - Log.w(TAG, "No " + codecName + " decoder available. Check the FFmpeg build configuration."); - return false; - } - return true; - } - - /** - * Returns whether the underlying library supports the specified MIME type. - * - * @param mimeType The MIME type to check. - */ - public static boolean supportsVideoFormat(String mimeType) { - if (!isAvailable()) { - return false; - } - String codecName = getVideoCodecName(mimeType); - if (codecName == null) { - return false; - } - if (!ffmpegHasDecoder(codecName)) { - Log.w(TAG, "No " + codecName + " decoder available. Check the FFmpeg build configuration."); - return false; - } - return true; - } - - /** - * Returns the name of the FFmpeg decoder that could be used to decode the format, or {@code null} - * if it's unsupported. - */ - /* package */ static @Nullable String getAudioCodecName(String mimeType) { - switch (mimeType) { - case MimeTypes.AUDIO_AAC: - return "aac"; - case MimeTypes.AUDIO_MPEG: - case MimeTypes.AUDIO_MPEG_L1: - case MimeTypes.AUDIO_MPEG_L2: - return "mp3"; - case MimeTypes.AUDIO_AC3: - return "ac3"; - case MimeTypes.AUDIO_E_AC3: - case MimeTypes.AUDIO_E_AC3_JOC: - return "eac3"; - case MimeTypes.AUDIO_TRUEHD: - return "truehd"; - case MimeTypes.AUDIO_DTS: - case MimeTypes.AUDIO_DTS_HD: - return "dca"; - case MimeTypes.AUDIO_VORBIS: - return "vorbis"; - case MimeTypes.AUDIO_OPUS: - return "opus"; - case MimeTypes.AUDIO_AMR_NB: - return "amrnb"; - case MimeTypes.AUDIO_AMR_WB: - return "amrwb"; - case MimeTypes.AUDIO_FLAC: - return "flac"; - case MimeTypes.AUDIO_ALAC: - return "alac"; - case MimeTypes.AUDIO_MLAW: - return "pcm_mulaw"; - case MimeTypes.AUDIO_ALAW: - return "pcm_alaw"; - default: - return null; - } - } - - /** - * Returns the name of the FFmpeg decoder that could be used to decode the format, or {@code null} - * if it's unsupported. - */ - /* package */ static @Nullable String getVideoCodecName(String mimeType) { - switch (mimeType) { - case MimeTypes.VIDEO_H264: - return "h264"; - case MimeTypes.VIDEO_H265: - return "hevc"; - default: - return null; - } - } - - private static native String ffmpegGetVersion(); - private static native boolean ffmpegHasDecoder(String codecName); - -} diff --git a/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegVideoDecoderException.java b/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegVideoDecoderException.java deleted file mode 100644 index 164c183ea44..00000000000 --- a/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegVideoDecoderException.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.android.exoplayer2.ext.ffmpeg; - -import com.google.android.exoplayer2.video.VideoDecoderException; - -/** Thrown when a libgav1 decoder error occurs. */ -public final class FfmpegVideoDecoderException extends VideoDecoderException { - - /* package */ FfmpegVideoDecoderException(String message) { - super(message); - } - - /* package */ FfmpegVideoDecoderException(String message, Throwable cause) { - super(message, cause); - } -} diff --git a/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegVideoRenderer.java b/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegVideoRenderer.java deleted file mode 100644 index 770a6a5fd21..00000000000 --- a/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegVideoRenderer.java +++ /dev/null @@ -1,190 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package com.google.android.exoplayer2.ext.ffmpeg; - -import static java.lang.Runtime.getRuntime; - -import android.os.Handler; -import android.view.Surface; -import androidx.annotation.Keep; -import androidx.annotation.Nullable; -import com.google.android.exoplayer2.C; -import com.google.android.exoplayer2.ExoPlaybackException; -import com.google.android.exoplayer2.ExoPlayer; -import com.google.android.exoplayer2.Format; -import com.google.android.exoplayer2.PlayerMessage.Target; -import com.google.android.exoplayer2.RendererCapabilities; -import com.google.android.exoplayer2.decoder.SimpleDecoder; -import com.google.android.exoplayer2.drm.ExoMediaCrypto; -import com.google.android.exoplayer2.util.TraceUtil; -import com.google.android.exoplayer2.util.Util; -import com.google.android.exoplayer2.video.SimpleDecoderVideoRenderer; -import com.google.android.exoplayer2.video.VideoDecoderException; -import com.google.android.exoplayer2.video.VideoDecoderInputBuffer; -import com.google.android.exoplayer2.video.VideoDecoderOutputBuffer; -import com.google.android.exoplayer2.video.VideoDecoderOutputBufferRenderer; -import com.google.android.exoplayer2.video.VideoRendererEventListener; - -/** - * Decodes and renders video using libgav1 decoder. - * - *

This renderer accepts the following messages sent via {@link ExoPlayer#createMessage(Target)} - * on the playback thread: - * - *

    - *
  • Message with type {@link #MSG_SET_SURFACE} to set the output surface. The message payload - * should be the target {@link Surface}, or null. - *
  • Message with type {@link #MSG_SET_VIDEO_DECODER_OUTPUT_BUFFER_RENDERER} to set the output - * buffer renderer. The message payload should be the target {@link - * VideoDecoderOutputBufferRenderer}, or null. - *
- */ -@Keep -public class FfmpegVideoRenderer extends SimpleDecoderVideoRenderer { - - private static final int DEFAULT_NUM_OF_INPUT_BUFFERS = 4; - private static final int DEFAULT_NUM_OF_OUTPUT_BUFFERS = 4; - /* Default size based on 720p resolution video compressed by a factor of two. */ - private static final int DEFAULT_INPUT_BUFFER_SIZE = - Util.ceilDivide(1280, 64) * Util.ceilDivide(720, 64) * (64 * 64 * 3 / 2) / 2; - - /** The number of input buffers. */ - private final int numInputBuffers; - /** - * The number of output buffers. The renderer may limit the minimum possible value due to - * requiring multiple output buffers to be dequeued at a time for it to make progress. - */ - private final int numOutputBuffers; - - private final int threads; - - @Nullable private FfmpegVideoDecoder decoder; - - /** - * Creates a Libgav1VideoRenderer. - * - * @param allowedJoiningTimeMs The maximum duration in milliseconds for which this video renderer - * can attempt to seamlessly join an ongoing playback. - * @param eventHandler A handler to use when delivering events to {@code eventListener}. May be - * null if delivery of events is not required. - * @param eventListener A listener of events. May be null if delivery of events is not required. - * @param maxDroppedFramesToNotify The maximum number of frames that can be dropped between - * invocations of {@link VideoRendererEventListener#onDroppedFrames(int, long)}. - */ - public FfmpegVideoRenderer( - long allowedJoiningTimeMs, - @Nullable Handler eventHandler, - @Nullable VideoRendererEventListener eventListener, - int maxDroppedFramesToNotify) { - this( - allowedJoiningTimeMs, - eventHandler, - eventListener, - maxDroppedFramesToNotify, - /* threads= */ getRuntime().availableProcessors(), - DEFAULT_NUM_OF_INPUT_BUFFERS, - DEFAULT_NUM_OF_OUTPUT_BUFFERS); - } - - /** - * Creates a Libgav1VideoRenderer. - * - * @param allowedJoiningTimeMs The maximum duration in milliseconds for which this video renderer - * can attempt to seamlessly join an ongoing playback. - * @param eventHandler A handler to use when delivering events to {@code eventListener}. May be - * null if delivery of events is not required. - * @param eventListener A listener of events. May be null if delivery of events is not required. - * @param maxDroppedFramesToNotify The maximum number of frames that can be dropped between - * invocations of {@link VideoRendererEventListener#onDroppedFrames(int, long)}. - * @param threads Number of threads libgav1 will use to decode. - * @param numInputBuffers Number of input buffers. - * @param numOutputBuffers Number of output buffers. - */ - public FfmpegVideoRenderer( - long allowedJoiningTimeMs, - @Nullable Handler eventHandler, - @Nullable VideoRendererEventListener eventListener, - int maxDroppedFramesToNotify, - int threads, - int numInputBuffers, - int numOutputBuffers) { - super(allowedJoiningTimeMs, eventHandler, eventListener, maxDroppedFramesToNotify); - this.threads = threads; - this.numInputBuffers = numInputBuffers; - this.numOutputBuffers = numOutputBuffers; - } - - @Override - @Capabilities - public final int supportsFormat(Format format) { - if (!FfmpegLibrary.isAvailable() - || !FfmpegLibrary.supportsVideoFormat(format.sampleMimeType)) { - return RendererCapabilities.create(FORMAT_UNSUPPORTED_TYPE); - } - if (format.drmInitData != null && format.exoMediaCryptoType == null) { - return RendererCapabilities.create(FORMAT_UNSUPPORTED_DRM); - } - return RendererCapabilities.create(FORMAT_HANDLED, ADAPTIVE_SEAMLESS, TUNNELING_NOT_SUPPORTED); - } - - @Override - protected SimpleDecoder< - VideoDecoderInputBuffer, - ? extends VideoDecoderOutputBuffer, - ? extends VideoDecoderException> - createDecoder(Format format, @Nullable ExoMediaCrypto mediaCrypto) - throws VideoDecoderException { - TraceUtil.beginSection("createGav1Decoder"); - int initialInputBufferSize = - format.maxInputSize != Format.NO_VALUE ? format.maxInputSize : DEFAULT_INPUT_BUFFER_SIZE; - FfmpegVideoDecoder decoder = - new FfmpegVideoDecoder(numInputBuffers, numOutputBuffers, initialInputBufferSize, threads, format); - this.decoder = decoder; - TraceUtil.endSection(); - return decoder; - } - - @Override - protected void renderOutputBufferToSurface(VideoDecoderOutputBuffer outputBuffer, Surface surface) - throws FfmpegVideoDecoderException { - if (decoder == null) { - throw new FfmpegVideoDecoderException( - "Failed to render output buffer to surface: decoder is not initialized."); - } - decoder.renderToSurface(outputBuffer, surface); - outputBuffer.release(); - } - - @Override - protected void setDecoderOutputMode(@C.VideoOutputMode int outputMode) { - if (decoder != null) { - decoder.setOutputMode(outputMode); - } - } - - // PlayerMessage.Target implementation. - - @Override - public void handleMessage(int messageType, @Nullable Object message) throws ExoPlaybackException { - if (messageType == MSG_SET_SURFACE) { - setOutputSurface((Surface) message); - } else if (messageType == MSG_SET_VIDEO_DECODER_OUTPUT_BUFFER_RENDERER) { - setOutputBufferRenderer((VideoDecoderOutputBufferRenderer) message); - } else { - super.handleMessage(messageType, message); - } - } -} diff --git a/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/package-info.java b/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/package-info.java deleted file mode 100644 index a9fedb19cb6..00000000000 --- a/extensions/ffmpegvideo/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/package-info.java +++ /dev/null @@ -1,19 +0,0 @@ -/* - * Copyright (C) 2019 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -@NonNullApi -package com.google.android.exoplayer2.ext.ffmpeg; - -import com.google.android.exoplayer2.util.NonNullApi; diff --git a/extensions/ffmpegvideo/src/main/jni/ffmpeg_audio_jni.cc b/extensions/ffmpegvideo/src/main/jni/ffmpeg_audio_jni.cc deleted file mode 100644 index 89dc2d3de06..00000000000 --- a/extensions/ffmpegvideo/src/main/jni/ffmpeg_audio_jni.cc +++ /dev/null @@ -1,360 +0,0 @@ -/* - * Copyright (C) 2016 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include -#include -#include - -extern "C" { -#ifdef __cplusplus -#define __STDC_CONSTANT_MACROS -#ifdef _STDINT_H -#undef _STDINT_H -#endif -#include -#endif -#include -#include -#include -#include -#include -} - -#define LOG_TAG "ffmpeg_jni" -#define LOGE(...) ((void)__android_log_print(ANDROID_LOG_ERROR, LOG_TAG, \ - __VA_ARGS__)) - -#define DECODER_FUNC(RETURN_TYPE, NAME, ...) \ - extern "C" { \ - JNIEXPORT RETURN_TYPE \ - Java_com_google_android_exoplayer2_ext_ffmpeg_FfmpegAudioDecoder_ ## NAME \ - (JNIEnv* env, jobject thiz, ##__VA_ARGS__);\ - } \ - JNIEXPORT RETURN_TYPE \ - Java_com_google_android_exoplayer2_ext_ffmpeg_FfmpegAudioDecoder_ ## NAME \ - (JNIEnv* env, jobject thiz, ##__VA_ARGS__)\ - -#define LIBRARY_FUNC(RETURN_TYPE, NAME, ...) \ - extern "C" { \ - JNIEXPORT RETURN_TYPE \ - Java_com_google_android_exoplayer2_ext_ffmpeg_FfmpegLibrary_ ## NAME \ - (JNIEnv* env, jobject thiz, ##__VA_ARGS__);\ - } \ - JNIEXPORT RETURN_TYPE \ - Java_com_google_android_exoplayer2_ext_ffmpeg_FfmpegLibrary_ ## NAME \ - (JNIEnv* env, jobject thiz, ##__VA_ARGS__)\ - -#define ERROR_STRING_BUFFER_LENGTH 256 - -// Output format corresponding to AudioFormat.ENCODING_PCM_16BIT. -static const AVSampleFormat OUTPUT_FORMAT_PCM_16BIT = AV_SAMPLE_FMT_S16; -// Output format corresponding to AudioFormat.ENCODING_PCM_FLOAT. -static const AVSampleFormat OUTPUT_FORMAT_PCM_FLOAT = AV_SAMPLE_FMT_FLT; - -// Error codes matching FfmpegAudioDecoder.java. -static const int DECODER_ERROR_INVALID_DATA = -1; -static const int DECODER_ERROR_OTHER = -2; - -/** - * Returns the AVCodec with the specified name, or NULL if it is not available. - */ -AVCodec *getCodecByName(JNIEnv* env, jstring codecName); - -/** - * Allocates and opens a new AVCodecContext for the specified codec, passing the - * provided extraData as initialization data for the decoder if it is non-NULL. - * Returns the created context. - */ -AVCodecContext *createContext(JNIEnv *env, AVCodec *codec, jbyteArray extraData, - jboolean outputFloat, jint rawSampleRate, - jint rawChannelCount); - -/** - * Decodes the packet into the output buffer, returning the number of bytes - * written, or a negative DECODER_ERROR constant value in the case of an error. - */ -int decodePacket(AVCodecContext *context, AVPacket *packet, - uint8_t *outputBuffer, int outputSize); - -/** - * Outputs a log message describing the avcodec error number. - */ -void logError(const char *functionName, int errorNumber); - -/** - * Releases the specified context. - */ -void releaseContext(AVCodecContext *context); - -jint JNI_OnLoad(JavaVM *vm, void *reserved) { - JNIEnv *env; - if (vm->GetEnv(reinterpret_cast(&env), JNI_VERSION_1_6) != JNI_OK) { - return -1; - } - avcodec_register_all(); - return JNI_VERSION_1_6; -} - -LIBRARY_FUNC(jstring, ffmpegGetVersion) { - return env->NewStringUTF(LIBAVCODEC_IDENT); -} - -LIBRARY_FUNC(jboolean, ffmpegHasDecoder, jstring codecName) { - return getCodecByName(env, codecName) != NULL; -} - -DECODER_FUNC(jlong, ffmpegInitialize, jstring codecName, jbyteArray extraData, - jboolean outputFloat, jint rawSampleRate, jint rawChannelCount) { - AVCodec *codec = getCodecByName(env, codecName); - if (!codec) { - LOGE("Codec not found."); - return 0L; - } - return (jlong)createContext(env, codec, extraData, outputFloat, rawSampleRate, - rawChannelCount); -} - -DECODER_FUNC(jint, ffmpegDecode, jlong context, jobject inputData, - jint inputSize, jobject outputData, jint outputSize) { - if (!context) { - LOGE("Context must be non-NULL."); - return -1; - } - if (!inputData || !outputData) { - LOGE("Input and output buffers must be non-NULL."); - return -1; - } - if (inputSize < 0) { - LOGE("Invalid input buffer size: %d.", inputSize); - return -1; - } - if (outputSize < 0) { - LOGE("Invalid output buffer length: %d", outputSize); - return -1; - } - uint8_t *inputBuffer = (uint8_t *) env->GetDirectBufferAddress(inputData); - uint8_t *outputBuffer = (uint8_t *) env->GetDirectBufferAddress(outputData); - AVPacket packet; - av_init_packet(&packet); - packet.data = inputBuffer; - packet.size = inputSize; - return decodePacket((AVCodecContext *) context, &packet, outputBuffer, - outputSize); -} - -DECODER_FUNC(jint, ffmpegGetChannelCount, jlong context) { - if (!context) { - LOGE("Context must be non-NULL."); - return -1; - } - return ((AVCodecContext *) context)->channels; -} - -DECODER_FUNC(jint, ffmpegGetSampleRate, jlong context) { - if (!context) { - LOGE("Context must be non-NULL."); - return -1; - } - return ((AVCodecContext *) context)->sample_rate; -} - -DECODER_FUNC(jlong, ffmpegReset, jlong jContext, jbyteArray extraData) { - AVCodecContext *context = (AVCodecContext *) jContext; - if (!context) { - LOGE("Tried to reset without a context."); - return 0L; - } - - AVCodecID codecId = context->codec_id; - if (codecId == AV_CODEC_ID_TRUEHD) { - // Release and recreate the context if the codec is TrueHD. - // TODO: Figure out why flushing doesn't work for this codec. - releaseContext(context); - AVCodec *codec = avcodec_find_decoder(codecId); - if (!codec) { - LOGE("Unexpected error finding codec %d.", codecId); - return 0L; - } - jboolean outputFloat = - (jboolean)(context->request_sample_fmt == OUTPUT_FORMAT_PCM_FLOAT); - return (jlong)createContext(env, codec, extraData, outputFloat, - /* rawSampleRate= */ -1, - /* rawChannelCount= */ -1); - } - - avcodec_flush_buffers(context); - return (jlong) context; -} - -DECODER_FUNC(void, ffmpegRelease, jlong context) { - if (context) { - releaseContext((AVCodecContext *) context); - } -} - -AVCodec *getCodecByName(JNIEnv* env, jstring codecName) { - if (!codecName) { - return NULL; - } - const char *codecNameChars = env->GetStringUTFChars(codecName, NULL); - AVCodec *codec = avcodec_find_decoder_by_name(codecNameChars); - env->ReleaseStringUTFChars(codecName, codecNameChars); - return codec; -} - -AVCodecContext *createContext(JNIEnv *env, AVCodec *codec, jbyteArray extraData, - jboolean outputFloat, jint rawSampleRate, - jint rawChannelCount) { - AVCodecContext *context = avcodec_alloc_context3(codec); - if (!context) { - LOGE("Failed to allocate context."); - return NULL; - } - context->request_sample_fmt = - outputFloat ? OUTPUT_FORMAT_PCM_FLOAT : OUTPUT_FORMAT_PCM_16BIT; - if (extraData) { - jsize size = env->GetArrayLength(extraData); - context->extradata_size = size; - context->extradata = - (uint8_t *) av_malloc(size + AV_INPUT_BUFFER_PADDING_SIZE); - if (!context->extradata) { - LOGE("Failed to allocate extradata."); - releaseContext(context); - return NULL; - } - env->GetByteArrayRegion(extraData, 0, size, (jbyte *) context->extradata); - } - if (context->codec_id == AV_CODEC_ID_PCM_MULAW || - context->codec_id == AV_CODEC_ID_PCM_ALAW) { - context->sample_rate = rawSampleRate; - context->channels = rawChannelCount; - context->channel_layout = av_get_default_channel_layout(rawChannelCount); - } - context->err_recognition = AV_EF_IGNORE_ERR; - int result = avcodec_open2(context, codec, NULL); - if (result < 0) { - logError("avcodec_open2", result); - releaseContext(context); - return NULL; - } - return context; -} - -int decodePacket(AVCodecContext *context, AVPacket *packet, - uint8_t *outputBuffer, int outputSize) { - int result = 0; - // Queue input data. - result = avcodec_send_packet(context, packet); - if (result) { - logError("avcodec_send_packet", result); - return result == AVERROR_INVALIDDATA ? DECODER_ERROR_INVALID_DATA - : DECODER_ERROR_OTHER; - } - - // Dequeue output data until it runs out. - int outSize = 0; - while (true) { - AVFrame *frame = av_frame_alloc(); - if (!frame) { - LOGE("Failed to allocate output frame."); - return -1; - } - result = avcodec_receive_frame(context, frame); - if (result) { - av_frame_free(&frame); - if (result == AVERROR(EAGAIN)) { - break; - } - logError("avcodec_receive_frame", result); - return result; - } - - // Resample output. - AVSampleFormat sampleFormat = context->sample_fmt; - int channelCount = context->channels; - int channelLayout = context->channel_layout; - int sampleRate = context->sample_rate; - int sampleCount = frame->nb_samples; - int dataSize = av_samples_get_buffer_size(NULL, channelCount, sampleCount, - sampleFormat, 1); - SwrContext *resampleContext; - if (context->opaque) { - resampleContext = (SwrContext *)context->opaque; - } else { - resampleContext = swr_alloc(); - av_opt_set_int(resampleContext, "in_channel_layout", channelLayout, 0); - av_opt_set_int(resampleContext, "out_channel_layout", channelLayout, 0); - av_opt_set_int(resampleContext, "in_sample_rate", sampleRate, 0); - av_opt_set_int(resampleContext, "out_sample_rate", sampleRate, 0); - av_opt_set_int(resampleContext, "in_sample_fmt", sampleFormat, 0); - // The output format is always the requested format. - av_opt_set_int(resampleContext, "out_sample_fmt", - context->request_sample_fmt, 0); - result = swr_init(resampleContext); - if (result < 0) { - logError("swr_init", result); - av_frame_free(&frame); - return -1; - } - context->opaque = resampleContext; - } - int inSampleSize = av_get_bytes_per_sample(sampleFormat); - int outSampleSize = av_get_bytes_per_sample(context->request_sample_fmt); - int outSamples = swr_get_out_samples(resampleContext, sampleCount); - int bufferOutSize = outSampleSize * channelCount * outSamples; - if (outSize + bufferOutSize > outputSize) { - LOGE("Output buffer size (%d) too small for output data (%d).", - outputSize, outSize + bufferOutSize); - av_frame_free(&frame); - return -1; - } - result = swr_convert(resampleContext, &outputBuffer, bufferOutSize, - (const uint8_t **)frame->data, frame->nb_samples); - av_frame_free(&frame); - if (result < 0) { - logError("swr_convert", result); - return result; - } - int available = swr_get_out_samples(resampleContext, 0); - if (available != 0) { - LOGE("Expected no samples remaining after resampling, but found %d.", - available); - return -1; - } - outputBuffer += bufferOutSize; - outSize += bufferOutSize; - } - return outSize; -} - -void logError(const char *functionName, int errorNumber) { - char *buffer = (char *) malloc(ERROR_STRING_BUFFER_LENGTH * sizeof(char)); - av_strerror(errorNumber, buffer, ERROR_STRING_BUFFER_LENGTH); - LOGE("Error in %s: %s", functionName, buffer); - free(buffer); -} - -void releaseContext(AVCodecContext *context) { - if (!context) { - return; - } - SwrContext *swrContext; - if ((swrContext = (SwrContext *)context->opaque)) { - swr_free(&swrContext); - context->opaque = NULL; - } - avcodec_free_context(&context); -} - diff --git a/extensions/ffmpegvideo/src/main/jni/ffmpeg_video_jni.cc b/extensions/ffmpegvideo/src/main/jni/ffmpeg_video_jni.cc deleted file mode 100644 index 2cb5a1c16d9..00000000000 --- a/extensions/ffmpegvideo/src/main/jni/ffmpeg_video_jni.cc +++ /dev/null @@ -1,436 +0,0 @@ -/* - * Copyright (C) 2016 The Android Open Source Project - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -#include -#include -#include -#include -#include -#include - -extern "C" { -#ifdef __cplusplus -#define __STDC_CONSTANT_MACROS -#ifdef _STDINT_H -#undef _STDINT_H -#endif - -#endif -#include -#include -#include -#include -#include -} - -#define LOG_TAG "ffmpeg_jni" -#define LOGE(...) ((void)__android_log_print(ANDROID_LOG_ERROR, LOG_TAG, \ - __VA_ARGS__)) - -#define DECODER_FUNC(RETURN_TYPE, NAME, ...) \ - extern "C" { \ - JNIEXPORT RETURN_TYPE \ - Java_com_google_android_exoplayer2_ext_ffmpeg_FfmpegVideoDecoder_ ## NAME \ - (JNIEnv* env, jobject thiz, ##__VA_ARGS__);\ - } \ - JNIEXPORT RETURN_TYPE \ - Java_com_google_android_exoplayer2_ext_ffmpeg_FfmpegVideoDecoder_ ## NAME \ - (JNIEnv* env, jobject thiz, ##__VA_ARGS__)\ - -#define ERROR_STRING_BUFFER_LENGTH 256 - - -namespace { -// Error codes matching FfmpegAudioDecoder.java. -const int DECODER_SUCCESS = 0; -const int DECODER_ERROR_INVALID_DATA = -1; -const int DECODER_ERROR_OTHER = -2; -const int DECODER_ERROR_READ_FRAME = -3; -const int DECODER_ERROR_SEND_PACKET = -4; - -// YUV plane indices. -const int kPlaneY = 0; -const int kPlaneU = 1; -const int kPlaneV = 2; -const int kMaxPlanes = 3; - -// Android YUV format. See: -// https://developer.android.com/reference/android/graphics/ImageFormat.html#YV12. -const int kImageFormatYV12 = 0x32315659; - -// LINT.IfChange -// Output modes. -const int kOutputModeYuv = 0; -const int kOutputModeSurfaceYuv = 1; -// LINT.ThenChange(../../../../../library/core/src/main/java/com/google/android/exoplayer2/C.java) - -// LINT.IfChange -const int kColorSpaceUnknown = 0; -// LINT.ThenChange(../../../../../library/core/src/main/java/com/google/android/exoplayer2/video/VideoDecoderOutputBuffer.java) - -struct JniContext { - ~JniContext() { - if (native_window) { - ANativeWindow_release(native_window); - } - } - - bool MaybeAcquireNativeWindow(JNIEnv *env, jobject new_surface) { - if (surface == new_surface) { - return true; - } - if (native_window) { - ANativeWindow_release(native_window); - } - native_window_width = 0; - native_window_height = 0; - native_window = ANativeWindow_fromSurface(env, new_surface); - if (native_window == nullptr) { - LOGE("kJniStatusANativeWindowError"); - surface = nullptr; - return false; - } - surface = new_surface; - return true; - } - - jfieldID data_field; - jfieldID yuvPlanes_field; - jfieldID yuvStrides_field; - jmethodID init_for_private_frame_method; - jmethodID init_for_yuv_frame_method; - jmethodID init_method; - - AVCodecContext *codecContext; - - ANativeWindow *native_window = nullptr; - jobject surface = nullptr; - int native_window_width = 0; - int native_window_height = 0; -}; - - -AVCodec *getCodecByName(JNIEnv *env, jstring codecName) { - if (!codecName) { - return NULL; - } - const char *codecNameChars = env->GetStringUTFChars(codecName, NULL); - AVCodec *codec = avcodec_find_decoder_by_name(codecNameChars); - env->ReleaseStringUTFChars(codecName, codecNameChars); - return codec; -} - -void logError(const char *functionName, int errorNumber) { - char *buffer = (char *) malloc(ERROR_STRING_BUFFER_LENGTH * sizeof(char)); - av_strerror(errorNumber, buffer, ERROR_STRING_BUFFER_LENGTH); - LOGE("Error in %s: %s", functionName, buffer); - free(buffer); -} - -void releaseContext(AVCodecContext *context) { - if (!context) { - return; - } - - avcodec_free_context(&context); -} - -JniContext *createContext(JNIEnv *env, - AVCodec *codec, - jbyteArray extraData, - jint threads) { - JniContext *jniContext = new(std::nothrow) JniContext(); - - AVCodecContext *codecContext = avcodec_alloc_context3(codec); - if (!codecContext) { - LOGE("Failed to allocate context."); - return NULL; - } - - if (extraData) { - jsize size = env->GetArrayLength(extraData); - codecContext->extradata_size = size; - codecContext->extradata = - (uint8_t *) av_malloc(size + AV_INPUT_BUFFER_PADDING_SIZE); - if (!codecContext->extradata) { - LOGE("Failed to allocate extradata."); - releaseContext(codecContext); - return NULL; - } - env->GetByteArrayRegion(extraData, 0, size, (jbyte *) codecContext->extradata); - } - - codecContext->thread_count = threads; - codecContext->err_recognition = AV_EF_IGNORE_ERR; - int result = avcodec_open2(codecContext, codec, NULL); - if (result < 0) { - logError("avcodec_open2", result); - releaseContext(codecContext); - return NULL; - } - - jniContext->codecContext = codecContext; - - // Populate JNI References. - const jclass outputBufferClass = env->FindClass( - "com/google/android/exoplayer2/video/VideoDecoderOutputBuffer"); - jniContext->data_field = env->GetFieldID(outputBufferClass, "data", "Ljava/nio/ByteBuffer;"); - jniContext->yuvPlanes_field = - env->GetFieldID(outputBufferClass, "yuvPlanes", "[Ljava/nio/ByteBuffer;"); - jniContext->yuvStrides_field = env->GetFieldID(outputBufferClass, "yuvStrides", "[I"); - jniContext->init_for_private_frame_method = - env->GetMethodID(outputBufferClass, "initForPrivateFrame", "(II)V"); - jniContext->init_for_yuv_frame_method = - env->GetMethodID(outputBufferClass, "initForYuvFrame", "(IIIII)Z"); - jniContext->init_method = - env->GetMethodID(outputBufferClass, "init", "(JILjava/nio/ByteBuffer;)V"); - - return jniContext; -} - -void CopyPlane(const uint8_t *source, int source_stride, uint8_t *destination, - int destination_stride, int width, int height) { - while (height--) { - std::memcpy(destination, source, width); - source += source_stride; - destination += destination_stride; - } -} - -constexpr int AlignTo16(int value) { return (value + 15) & (~15); } - -} - -DECODER_FUNC(jlong, ffmpegInitialize, jstring codecName, jbyteArray extraData, jint threads) { - AVCodec *codec = getCodecByName(env, codecName); - if (!codec) { - LOGE("Codec not found."); - return 0L; - } - - return (jlong) createContext(env, codec, extraData, threads); -} - -DECODER_FUNC(jlong, ffmpegReset, jlong jContext) { - JniContext *const jniContext = reinterpret_cast(jContext); - AVCodecContext *context = jniContext->codecContext; - if (!context) { - LOGE("Tried to reset without a context."); - return 0L; - } - -// LOGE("avcodec_flush_buffers"); - avcodec_flush_buffers(context); - return (jlong) jniContext; -} - -DECODER_FUNC(void, ffmpegRelease, jlong jContext) { - JniContext *const jniContext = reinterpret_cast(jContext); - AVCodecContext *context = jniContext->codecContext; - if (context) { - releaseContext(context); - } -} - - -DECODER_FUNC(jint, ffmpegSendPacket, jlong jContext, jobject encodedData, - jint length, jlong inputTimeUs) { - JniContext *const jniContext = reinterpret_cast(jContext); - AVCodecContext *avContext = jniContext->codecContext; - - uint8_t *inputBuffer = (uint8_t *) env->GetDirectBufferAddress(encodedData); - AVPacket packet; - av_init_packet(&packet); - packet.data = inputBuffer; - packet.size = length; - packet.pts = inputTimeUs; - - int result = 0; - // Queue input data. - result = avcodec_send_packet(avContext, &packet); - if (result) { - logError("avcodec_send_packet", result); - if (result == AVERROR_INVALIDDATA) { - // need more data - return DECODER_ERROR_INVALID_DATA; - } else if (result == AVERROR(EAGAIN)) { - // need read frame - return DECODER_ERROR_READ_FRAME; - } else { - return DECODER_ERROR_OTHER; - } - } - return result; -} - -DECODER_FUNC(jint, ffmpegReceiveFrame, jlong jContext, jint outputMode, jobject jOutputBuffer, - jboolean decodeOnly) { - JniContext *const jniContext = reinterpret_cast(jContext); - AVCodecContext *avContext = jniContext->codecContext; - int result = 0; - - AVFrame *frame = av_frame_alloc(); - if (!frame) { - LOGE("Failed to allocate output frame."); - return DECODER_ERROR_OTHER; - } - result = avcodec_receive_frame(avContext, frame); - - // fail - if (decodeOnly || result == AVERROR(EAGAIN)) { - // This is not an error. The input data was decode-only or no displayable - // frames are available. - av_frame_free(&frame); - return DECODER_ERROR_INVALID_DATA; - } - if (result) { - av_frame_free(&frame); - logError("avcodec_receive_frame", result); - return DECODER_ERROR_OTHER; - } - - // success - // init time and mode - env->CallVoidMethod(jOutputBuffer, jniContext->init_method, frame->pts, outputMode, nullptr); - - // init data - const jboolean init_result = env->CallBooleanMethod( - jOutputBuffer, jniContext->init_for_yuv_frame_method, - frame->width, - frame->height, - frame->linesize[0], frame->linesize[1], - 0); - if (env->ExceptionCheck()) { - // Exception is thrown in Java when returning from the native call. - return DECODER_ERROR_OTHER; - } - if (!init_result) { - return DECODER_ERROR_OTHER; - } - - const jobject data_object = env->GetObjectField(jOutputBuffer, jniContext->data_field); - jbyte *data = reinterpret_cast(env->GetDirectBufferAddress(data_object)); - const int32_t uvHeight = (frame->height + 1) / 2; - const uint64_t yLength = frame->linesize[0] * frame->height; - const uint64_t uvLength = frame->linesize[1] * uvHeight; - - // todo rotate YUV data - - memcpy(data, frame->data[0], yLength); - memcpy(data + yLength, frame->data[1], uvLength); - memcpy(data + yLength + uvLength, frame->data[2], uvLength); - - av_frame_free(&frame); - - return result; -} - -DECODER_FUNC(jint, ffmpegRenderFrame, jlong jContext, jobject jSurface, - jobject jOutputBuffer, jint displayedWidth, jint displayedHeight) { - JniContext *const jniContext = reinterpret_cast(jContext); - if (!jniContext->MaybeAcquireNativeWindow(env, jSurface)) { - return DECODER_ERROR_OTHER; - } - - if (jniContext->native_window_width != displayedWidth || - jniContext->native_window_height != displayedHeight) { - if (ANativeWindow_setBuffersGeometry( - jniContext->native_window, - displayedWidth, - displayedHeight, - kImageFormatYV12)) { - LOGE("kJniStatusANativeWindowError"); - return DECODER_ERROR_OTHER; - } - jniContext->native_window_width = displayedWidth; - jniContext->native_window_height = displayedHeight; - } - - ANativeWindow_Buffer native_window_buffer; - if (ANativeWindow_lock(jniContext->native_window, &native_window_buffer, - /*inOutDirtyBounds=*/nullptr) || - native_window_buffer.bits == nullptr) { - LOGE("kJniStatusANativeWindowError"); - return DECODER_ERROR_OTHER; - } - - jobject yuvPlanes_object = env->GetObjectField(jOutputBuffer, jniContext->yuvPlanes_field); - jobjectArray yuvPlanes_array = static_cast(yuvPlanes_object); - jobject yuvPlanesY = env->GetObjectArrayElement(yuvPlanes_array, kPlaneY); - jobject yuvPlanesU = env->GetObjectArrayElement(yuvPlanes_array, kPlaneU); - jobject yuvPlanesV = env->GetObjectArrayElement(yuvPlanes_array, kPlaneV); - jbyte *planeY = reinterpret_cast(env->GetDirectBufferAddress(yuvPlanesY)); - jbyte *planeU = reinterpret_cast(env->GetDirectBufferAddress(yuvPlanesU)); - jbyte *planeV = reinterpret_cast(env->GetDirectBufferAddress(yuvPlanesV)); - - jobject yuvStrides_object = env->GetObjectField(jOutputBuffer, jniContext->yuvStrides_field); - jintArray *yuvStrides_array = reinterpret_cast(&yuvStrides_object); - - int *yuvStrides = env->GetIntArrayElements(*yuvStrides_array, NULL); - int strideY = yuvStrides[kPlaneY]; - int strideU = yuvStrides[kPlaneU]; - int strideV = yuvStrides[kPlaneV]; - - // Y plane - CopyPlane(reinterpret_cast(planeY), - strideY, - reinterpret_cast(native_window_buffer.bits), - native_window_buffer.stride, - displayedWidth, - displayedHeight); - - const int y_plane_size = - native_window_buffer.stride * native_window_buffer.height; - const int32_t native_window_buffer_uv_height = - (native_window_buffer.height + 1) / 2; - const int native_window_buffer_uv_stride = - AlignTo16(native_window_buffer.stride / 2); - - // TODO(b/140606738): Handle monochrome videos. - - // V plane - // Since the format for ANativeWindow is YV12, V plane is being processed - // before U plane. - const int v_plane_height = std::min(native_window_buffer_uv_height, - displayedHeight); - CopyPlane( - reinterpret_cast(planeV), - strideV, - reinterpret_cast(native_window_buffer.bits) + y_plane_size, - native_window_buffer_uv_stride, displayedWidth, - v_plane_height); - - const int v_plane_size = v_plane_height * native_window_buffer_uv_stride; - - // U plane - CopyPlane( - reinterpret_cast(planeU), - strideU, - reinterpret_cast(native_window_buffer.bits) + - y_plane_size + v_plane_size, - native_window_buffer_uv_stride, displayedWidth, - std::min(native_window_buffer_uv_height, - displayedHeight)); - - - env->ReleaseIntArrayElements(*yuvStrides_array, yuvStrides, 0); - - if (ANativeWindow_unlockAndPost(jniContext->native_window)) { - LOGE("kJniStatusANativeWindowError"); - return DECODER_ERROR_OTHER; - } - - return DECODER_SUCCESS; -} From 0e9bab2b257c5d58665abd0076e6e72aec139be5 Mon Sep 17 00:00:00 2001 From: haohao <358297604@qq.com> Date: Mon, 20 Apr 2020 14:32:15 +0800 Subject: [PATCH 3/5] Fix FfmpegVideoDecoder Lint.ThenChange paths --- .../ext/ffmpeg/FfmpegVideoDecoder.java | 40 ++++++++++--------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegVideoDecoder.java b/extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegVideoDecoder.java index 0e71f0fe051..90e04f05044 100644 --- a/extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegVideoDecoder.java +++ b/extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegVideoDecoder.java @@ -36,11 +36,13 @@ extends SimpleDecoder { - // Error codes matching ffmpeg_jni.cc. - private static final int DECODER_ERROR_INVALID_DATA = -1; - private static final int DECODER_ERROR_OTHER = -2; - private static final int DECODER_ERROR_READ_FRAME = -3; - private static final int DECODER_ERROR_SEND_PACKET = -4; + // LINT.IfChange + private static final int VIDEO_DECODER_SUCCESS = 0; + private static final int VIDEO_DECODER_ERROR_INVALID_DATA = -1; + private static final int VIDEO_DECODER_ERROR_OTHER = -2; + private static final int VIDEO_DECODER_ERROR_READ_FRAME = -3; + private static final int VIDEO_DECODER_ERROR_SEND_PACKET = -4; + // LINT.ThenChange(../../../../../../../jni/ffmpeg_jni.cc) private final String codecName; private long nativeContext; @@ -142,13 +144,13 @@ protected FfmpegDecoderException decode( boolean needSendAgain = false; int sendPacketResult = ffmpegSendPacket(nativeContext, inputData, inputSize, inputBuffer.timeUs); - if (sendPacketResult == DECODER_ERROR_INVALID_DATA) { + if (sendPacketResult == VIDEO_DECODER_ERROR_INVALID_DATA) { outputBuffer.setFlags(C.BUFFER_FLAG_DECODE_ONLY); return null; - } else if (sendPacketResult == DECODER_ERROR_READ_FRAME) { + } else if (sendPacketResult == VIDEO_DECODER_ERROR_READ_FRAME) { // need read frame needSendAgain = true; - } else if (sendPacketResult == DECODER_ERROR_OTHER) { + } else if (sendPacketResult == VIDEO_DECODER_ERROR_OTHER) { return new FfmpegDecoderException("ffmpegDecode error: (see logcat)"); } @@ -157,13 +159,13 @@ protected FfmpegDecoderException decode( // We need to dequeue the decoded frame from the decoder even when the input data is // decode-only. int getFrameResult = ffmpegReceiveFrame(nativeContext, outputMode, outputBuffer, decodeOnly); - if (getFrameResult == DECODER_ERROR_SEND_PACKET) { + if (getFrameResult == VIDEO_DECODER_ERROR_SEND_PACKET) { return null; - } else if (getFrameResult == DECODER_ERROR_OTHER) { + } else if (getFrameResult == VIDEO_DECODER_ERROR_OTHER) { return new FfmpegDecoderException("ffmpegDecode error: (see logcat)"); } - if (getFrameResult == DECODER_ERROR_INVALID_DATA) { + if (getFrameResult == VIDEO_DECODER_ERROR_INVALID_DATA) { outputBuffer.addFlag(C.BUFFER_FLAG_DECODE_ONLY); } @@ -206,7 +208,7 @@ public void renderToSurface(VideoDecoderOutputBuffer outputBuffer, Surface surfa } if (ffmpegRenderFrame( nativeContext, surface, - outputBuffer, outputBuffer.width, outputBuffer.height) == DECODER_ERROR_OTHER) { + outputBuffer, outputBuffer.width, outputBuffer.height) == VIDEO_DECODER_ERROR_OTHER) { throw new FfmpegDecoderException( "Buffer render error: "); } @@ -226,10 +228,11 @@ private native int ffmpegRenderFrame( /** * Decodes the encoded data passed. * - * @param context Decoder context. + * @param context Decoder context. * @param encodedData Encoded data. - * @param length Length of the data buffer. - * @return 0 if successful, {@link #DECODER_ERROR_OTHER} if an error occurred. + * @param length Length of the data buffer. + * @return {@link #VIDEO_DECODER_SUCCESS} if successful, {@link #VIDEO_DECODER_ERROR_OTHER} if an + * error occurred. */ private native int ffmpegSendPacket(long context, ByteBuffer encodedData, int length, long inputTime); @@ -237,10 +240,11 @@ private native int ffmpegSendPacket(long context, ByteBuffer encodedData, int le /** * Gets the decoded frame. * - * @param context Decoder context. + * @param context Decoder context. * @param outputBuffer Output buffer for the decoded frame. - * @return 0 if successful, {@link #DECODER_ERROR_INVALID_DATA} if successful but the frame is - * decode-only, {@link #DECODER_ERROR_OTHER} if an error occurred. + * @return {@link #VIDEO_DECODER_SUCCESS} if successful, {@link #VIDEO_DECODER_ERROR_INVALID_DATA} + * if successful but the frame is decode-only, {@link #VIDEO_DECODER_ERROR_OTHER} if an error + * occurred. */ private native int ffmpegReceiveFrame( long context, int outputMode, VideoDecoderOutputBuffer outputBuffer, boolean decodeOnly); From 37ef691a0d1f81c9100f5f86a79f2dae80587131 Mon Sep 17 00:00:00 2001 From: haohao <358297604@qq.com> Date: Mon, 20 Apr 2020 15:46:40 +0800 Subject: [PATCH 4/5] Use the appropriate method to link to Ffmpeg Library --- .../exoplayer2/ext/ffmpeg/FfmpegLibrary.java | 4 +- .../ext/ffmpeg/FfmpegVideoRenderer.java | 2 + extensions/ffmpeg/src/main/jni/CMakeLists.txt | 61 +++++++++---------- extensions/ffmpeg/src/main/jni/ffmpeg_jni.cc | 2 - 4 files changed, 32 insertions(+), 37 deletions(-) diff --git a/extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegLibrary.java b/extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegLibrary.java index c25bdec8132..4b0d642e252 100644 --- a/extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegLibrary.java +++ b/extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegLibrary.java @@ -32,10 +32,8 @@ public final class FfmpegLibrary { private static final String TAG = "FfmpegLibrary"; - // TODO: Use appropriate compilation script. private static final LibraryLoader LOADER = - new LibraryLoader("ffmpeg", "ffmpegJNI"); -// new LibraryLoader("avutil", "swresample", "avcodec", "ffmpeg"); + new LibraryLoader("avutil", "swresample", "avcodec", "ffmpegJNI"); private FfmpegLibrary() {} diff --git a/extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegVideoRenderer.java b/extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegVideoRenderer.java index 94458eba001..e1071b9fef5 100644 --- a/extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegVideoRenderer.java +++ b/extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegVideoRenderer.java @@ -42,6 +42,8 @@ */ public final class FfmpegVideoRenderer extends DecoderVideoRenderer { + private static final String TAG = "FfmpegVideoRenderer"; + private static final int DEFAULT_NUM_OF_INPUT_BUFFERS = 4; private static final int DEFAULT_NUM_OF_OUTPUT_BUFFERS = 4; /* Default size based on 720p resolution video compressed by a factor of two. */ diff --git a/extensions/ffmpeg/src/main/jni/CMakeLists.txt b/extensions/ffmpeg/src/main/jni/CMakeLists.txt index 0d645ec6221..da92feaf3bd 100644 --- a/extensions/ffmpeg/src/main/jni/CMakeLists.txt +++ b/extensions/ffmpeg/src/main/jni/CMakeLists.txt @@ -6,40 +6,37 @@ set(CMAKE_CXX_STANDARD 11) project(libffmpegJNI C CXX) -# Devices using armeabi-v7a are not required to support -# Neon which is why Neon is disabled by default for -# armeabi-v7a build. This flag enables it. -if(${ANDROID_ABI} MATCHES "armeabi-v7a") - add_compile_options("-mfpu=neon") - add_compile_options("-fPIC") -endif() - set(libgffmpeg_jni_root "${CMAKE_CURRENT_SOURCE_DIR}") set(libgffmpeg_jni_build "${CMAKE_BINARY_DIR}") set(libgffmpeg_jni_output_directory ${libgffmpeg_jni_root}/../libs/${ANDROID_ABI}/) -#set(libgffmpeg_root "${libgffmpeg_jni_root}/libgffmpeg") -#set(libgffmpeg_build "${libgffmpeg_jni_build}/libgffmpeg") - -#set(cpu_features_root "${libgffmpeg_jni_root}/cpu_features") -#set(cpu_features_build "${libgffmpeg_jni_build}/cpu_features") - -# Build cpu_features library. -#add_subdirectory("${cpu_features_root}" -# "${cpu_features_build}" -# EXCLUDE_FROM_ALL) - -# Build libgffmpeg. -#add_subdirectory("${libgffmpeg_root}" -# "${libgffmpeg_build}" -# EXCLUDE_FROM_ALL) -add_library(ffmpeg - SHARED - IMPORTED) -set_target_properties(ffmpeg PROPERTIES - IMPORTED_LOCATION - ${libgffmpeg_jni_output_directory}/libffmpeg.so) +add_library( + avutil + SHARED + IMPORTED) +set_target_properties( + avutil PROPERTIES + IMPORTED_LOCATION + ${libgffmpeg_jni_output_directory}/libavutil.so) + +add_library( + swresample + SHARED + IMPORTED) +set_target_properties( + swresample PROPERTIES + IMPORTED_LOCATION + ${libgffmpeg_jni_output_directory}/libswresample.so) + +add_library( + avcodec + SHARED + IMPORTED) +set_target_properties( + avcodec PROPERTIES + IMPORTED_LOCATION + ${libgffmpeg_jni_output_directory}/libavcodec.so) # Build libgffmpegJNI. add_library(ffmpegJNI @@ -53,9 +50,9 @@ find_library(android_log_lib log) # Link libgffmpegJNI against used libraries. target_link_libraries(ffmpegJNI PRIVATE android - PRIVATE ffmpeg -# PRIVATE cpu_features -# PRIVATE libgffmpeg_static + PRIVATE avutil + PRIVATE swresample + PRIVATE avcodec PRIVATE ${android_log_lib}) # Specify output directory for libgffmpegJNI. diff --git a/extensions/ffmpeg/src/main/jni/ffmpeg_jni.cc b/extensions/ffmpeg/src/main/jni/ffmpeg_jni.cc index ce348cbffd4..c122fc38347 100644 --- a/extensions/ffmpeg/src/main/jni/ffmpeg_jni.cc +++ b/extensions/ffmpeg/src/main/jni/ffmpeg_jni.cc @@ -82,8 +82,6 @@ static const int AUDIO_DECODER_ERROR_INVALID_DATA = -1; static const int AUDIO_DECODER_ERROR_OTHER = -2; // LINT.ThenChange(../java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegAudioDecoder.java) -// Error codes matching FfmpegVideoDecoder.java. - // LINT.IfChange static const int VIDEO_DECODER_SUCCESS = 0; static const int VIDEO_DECODER_ERROR_INVALID_DATA = -1; From c80189f230073fbafa2a1cd355a8812d8a5b120d Mon Sep 17 00:00:00 2001 From: haohao <358297604@qq.com> Date: Wed, 6 May 2020 20:14:50 +0800 Subject: [PATCH 5/5] Fix ffmpeg library android.mk compilation error --- extensions/ffmpeg/build.gradle | 12 +++++----- .../exoplayer2/ext/ffmpeg/FfmpegLibrary.java | 2 +- .../ext/ffmpeg/FfmpegVideoDecoder.java | 14 ++++------- extensions/ffmpeg/src/main/jni/Android.mk | 2 +- extensions/ffmpeg/src/main/jni/CMakeLists.txt | 12 +++++----- extensions/ffmpeg/src/main/jni/ffmpeg_jni.cc | 24 +++++++------------ 6 files changed, 27 insertions(+), 39 deletions(-) diff --git a/extensions/ffmpeg/build.gradle b/extensions/ffmpeg/build.gradle index 83b653f06e9..4fe753427ac 100644 --- a/extensions/ffmpeg/build.gradle +++ b/extensions/ffmpeg/build.gradle @@ -32,7 +32,7 @@ android { // Debug CMake build type causes video frames to drop, // so native library should always use Release build type. arguments "-DCMAKE_BUILD_TYPE=Release" - targets "ffmpegJNI" + targets "ffmpeg" } } } @@ -52,13 +52,13 @@ android { } } - // This option resolves the problem of finding libffmpegJNI.so + // This option resolves the problem of finding libffmpeg.so // on multiple paths. The first one found is picked. packagingOptions { - pickFirst 'lib/arm64-v8a/libffmpegJNI.so' - pickFirst 'lib/armeabi-v7a/libffmpegJNI.so' - pickFirst 'lib/x86/libffmpegJNI.so' - pickFirst 'lib/x86_64/libffmpegJNI.so' + pickFirst 'lib/arm64-v8a/libffmpeg.so' + pickFirst 'lib/armeabi-v7a/libffmpeg.so' + pickFirst 'lib/x86/libffmpeg.so' + pickFirst 'lib/x86_64/libffmpeg.so' } sourceSets.main { diff --git a/extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegLibrary.java b/extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegLibrary.java index 4b0d642e252..e3752aad5c9 100644 --- a/extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegLibrary.java +++ b/extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegLibrary.java @@ -33,7 +33,7 @@ public final class FfmpegLibrary { private static final String TAG = "FfmpegLibrary"; private static final LibraryLoader LOADER = - new LibraryLoader("avutil", "swresample", "avcodec", "ffmpegJNI"); + new LibraryLoader("avutil", "swresample", "avcodec", "ffmpeg"); private FfmpegLibrary() {} diff --git a/extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegVideoDecoder.java b/extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegVideoDecoder.java index 90e04f05044..9eea7c62511 100644 --- a/extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegVideoDecoder.java +++ b/extensions/ffmpeg/src/main/java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegVideoDecoder.java @@ -36,12 +36,13 @@ extends SimpleDecoder { + private static final String TAG = "FfmpegVideoDecoder"; + // LINT.IfChange private static final int VIDEO_DECODER_SUCCESS = 0; private static final int VIDEO_DECODER_ERROR_INVALID_DATA = -1; private static final int VIDEO_DECODER_ERROR_OTHER = -2; private static final int VIDEO_DECODER_ERROR_READ_FRAME = -3; - private static final int VIDEO_DECODER_ERROR_SEND_PACKET = -4; // LINT.ThenChange(../../../../../../../jni/ffmpeg_jni.cc) private final String codecName; @@ -141,7 +142,6 @@ protected FfmpegDecoderException decode( ByteBuffer inputData = Util.castNonNull(inputBuffer.data); int inputSize = inputData.limit(); // enqueue origin data - boolean needSendAgain = false; int sendPacketResult = ffmpegSendPacket(nativeContext, inputData, inputSize, inputBuffer.timeUs); if (sendPacketResult == VIDEO_DECODER_ERROR_INVALID_DATA) { @@ -149,7 +149,7 @@ protected FfmpegDecoderException decode( return null; } else if (sendPacketResult == VIDEO_DECODER_ERROR_READ_FRAME) { // need read frame - needSendAgain = true; + Log.d(TAG, "VIDEO_DECODER_ERROR_READ_FRAME: " + "timeUs=" + inputBuffer.timeUs); } else if (sendPacketResult == VIDEO_DECODER_ERROR_OTHER) { return new FfmpegDecoderException("ffmpegDecode error: (see logcat)"); } @@ -159,9 +159,7 @@ protected FfmpegDecoderException decode( // We need to dequeue the decoded frame from the decoder even when the input data is // decode-only. int getFrameResult = ffmpegReceiveFrame(nativeContext, outputMode, outputBuffer, decodeOnly); - if (getFrameResult == VIDEO_DECODER_ERROR_SEND_PACKET) { - return null; - } else if (getFrameResult == VIDEO_DECODER_ERROR_OTHER) { + if (getFrameResult == VIDEO_DECODER_ERROR_OTHER) { return new FfmpegDecoderException("ffmpegDecode error: (see logcat)"); } @@ -173,10 +171,6 @@ protected FfmpegDecoderException decode( outputBuffer.colorInfo = inputBuffer.colorInfo; } - if (needSendAgain) { - Log.e("ffmpeg_jni", "timeUs=" + inputBuffer.timeUs + ", " + "nendSendAagin"); - } - return null; } diff --git a/extensions/ffmpeg/src/main/jni/Android.mk b/extensions/ffmpeg/src/main/jni/Android.mk index bcaf12cd119..01de61ccc1e 100644 --- a/extensions/ffmpeg/src/main/jni/Android.mk +++ b/extensions/ffmpeg/src/main/jni/Android.mk @@ -36,5 +36,5 @@ LOCAL_MODULE := ffmpeg LOCAL_SRC_FILES := ffmpeg_jni.cc LOCAL_C_INCLUDES := ffmpeg LOCAL_SHARED_LIBRARIES := libavcodec libswresample libavutil -LOCAL_LDLIBS := -Lffmpeg/android-libs/$(TARGET_ARCH_ABI) -llog +LOCAL_LDLIBS := -Lffmpeg/android-libs/$(TARGET_ARCH_ABI) -llog -landroid include $(BUILD_SHARED_LIBRARY) diff --git a/extensions/ffmpeg/src/main/jni/CMakeLists.txt b/extensions/ffmpeg/src/main/jni/CMakeLists.txt index da92feaf3bd..adab7d4f540 100644 --- a/extensions/ffmpeg/src/main/jni/CMakeLists.txt +++ b/extensions/ffmpeg/src/main/jni/CMakeLists.txt @@ -1,10 +1,10 @@ -# libgav1JNI requires modern CMake. +# libffmpegJNI requires modern CMake. cmake_minimum_required(VERSION 3.7.1 FATAL_ERROR) -# libgav1JNI requires C++11. +# libffmpegJNI requires C++11. set(CMAKE_CXX_STANDARD 11) -project(libffmpegJNI C CXX) +project(libffmpeg C CXX) set(libgffmpeg_jni_root "${CMAKE_CURRENT_SOURCE_DIR}") set(libgffmpeg_jni_build "${CMAKE_BINARY_DIR}") @@ -39,7 +39,7 @@ set_target_properties( ${libgffmpeg_jni_output_directory}/libavcodec.so) # Build libgffmpegJNI. -add_library(ffmpegJNI +add_library(ffmpeg SHARED ffmpeg_jni.cc) include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include) @@ -48,7 +48,7 @@ include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include) find_library(android_log_lib log) # Link libgffmpegJNI against used libraries. -target_link_libraries(ffmpegJNI +target_link_libraries(ffmpeg PRIVATE android PRIVATE avutil PRIVATE swresample @@ -56,6 +56,6 @@ target_link_libraries(ffmpegJNI PRIVATE ${android_log_lib}) # Specify output directory for libgffmpegJNI. -set_target_properties(ffmpegJNI PROPERTIES +set_target_properties(ffmpeg PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${libgffmpeg_jni_output_directory}) diff --git a/extensions/ffmpeg/src/main/jni/ffmpeg_jni.cc b/extensions/ffmpeg/src/main/jni/ffmpeg_jni.cc index c122fc38347..44d086e93f4 100644 --- a/extensions/ffmpeg/src/main/jni/ffmpeg_jni.cc +++ b/extensions/ffmpeg/src/main/jni/ffmpeg_jni.cc @@ -87,7 +87,6 @@ static const int VIDEO_DECODER_SUCCESS = 0; static const int VIDEO_DECODER_ERROR_INVALID_DATA = -1; static const int VIDEO_DECODER_ERROR_OTHER = -2; static const int VIDEO_DECODER_ERROR_READ_FRAME = -3; -static const int VIDEO_DECODER_ERROR_SEND_PACKET = -4; // LINT.ThenChange(../java/com/google/android/exoplayer2/ext/ffmpeg/FfmpegVideoDecoder.java) /** @@ -448,11 +447,6 @@ void CopyPlane(const uint8_t *source, int source_stride, uint8_t *destination, constexpr int AlignTo16(int value) { return (value + 15) & (~15); } -JniContext *createVideoContext(JNIEnv *env, - AVCodec *codec, - jbyteArray extraData, - jint threads); - JniContext *createVideoContext(JNIEnv *env, AVCodec *codec, jbyteArray extraData, @@ -651,9 +645,12 @@ VIDEO_DECODER_FUNC(jint, ffmpegRenderFrame, jlong jContext, jobject jSurface, } ANativeWindow_Buffer native_window_buffer; - if (ANativeWindow_lock(jniContext->native_window, &native_window_buffer, -/*inOutDirtyBounds=*/nullptr) || - native_window_buffer.bits == nullptr) { + int result = ANativeWindow_lock(jniContext->native_window, &native_window_buffer, nullptr); + if (result == -19) { + // Surface: dequeueBuffer failed (No such device) + jniContext->surface = nullptr; + return VIDEO_DECODER_SUCCESS; + } else if (result || native_window_buffer.bits == nullptr) { LOGE("kJniStatusANativeWindowError"); return VIDEO_DECODER_ERROR_OTHER; } @@ -683,12 +680,9 @@ VIDEO_DECODER_FUNC(jint, ffmpegRenderFrame, jlong jContext, jobject jSurface, displayedWidth, displayedHeight); - const int y_plane_size = - native_window_buffer.stride * native_window_buffer.height; - const int32_t native_window_buffer_uv_height = - (native_window_buffer.height + 1) / 2; - const int native_window_buffer_uv_stride = - AlignTo16(native_window_buffer.stride / 2); + const int y_plane_size = native_window_buffer.stride * native_window_buffer.height; + const int32_t native_window_buffer_uv_height = (native_window_buffer.height + 1) / 2; + const int native_window_buffer_uv_stride = AlignTo16(native_window_buffer.stride / 2); // TODO(b/140606738): Handle monochrome videos.