1 /*
2 * Copyright (C) 2019 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #include <source/FrameBufferSource.h>
18
19 #include <algorithm>
20 #include <chrono>
21
22 #include <libyuv/convert.h>
23
24 #include "host/libs/config/cuttlefish_config.h"
25
26 #include "vpx/vpx_encoder.h"
27 #include "vpx/vpx_codec.h"
28 #include "vpx/vp8cx.h"
29
30 #include <gflags/gflags.h>
31
32 #define ENABLE_LOGGING 0
33
34 namespace android {
35
36 namespace {
GetNowUs()37 int64_t GetNowUs() {
38 auto now = std::chrono::steady_clock::now().time_since_epoch();
39 return std::chrono::duration_cast<std::chrono::microseconds>(now).count();
40 }
41 }
42
43 struct FrameBufferSource::Encoder {
44 Encoder() = default;
45 virtual ~Encoder() = default;
46
47 virtual void forceIDRFrame() = 0;
48 virtual bool isForcingIDRFrame() const = 0;
49
50 virtual void storeFrame(const void* frame) = 0;
51 virtual std::shared_ptr<SBuffer> encodeStoredFrame(int64_t timeUs) = 0;
52 };
53
54 ////////////////////////////////////////////////////////////////////////////////
55
56 struct FrameBufferSource::VPXEncoder : public FrameBufferSource::Encoder {
57 VPXEncoder(int width, int height, int rateHz);
58 ~VPXEncoder() override;
59
60 void forceIDRFrame() override;
61 bool isForcingIDRFrame() const override;
62
63 void storeFrame(const void* frame) override;
64 std::shared_ptr<SBuffer> encodeStoredFrame(int64_t timeUs) override;
65
66 private:
67 int mWidth, mHeight, mRefreshRateHz;
68
69 int mSizeY, mSizeUV;
70 void *mI420Data;
71
72 vpx_codec_iface_t *mCodecInterface;
73 std::unique_ptr<vpx_codec_enc_cfg_t> mCodecConfiguration;
74
75 using contextFreeFunc = std::function<vpx_codec_err_t(vpx_codec_ctx_t *)>;
76 std::unique_ptr<vpx_codec_ctx_t, contextFreeFunc> mCodecContext;
77
78 std::atomic<bool> mForceIDRFrame;
79 bool mFirstFrame;
80 bool mStoredFrame;
81 int64_t mLastTimeUs;
82 };
83
GetCPUCoreCount()84 static int GetCPUCoreCount() {
85 int cpuCoreCount;
86
87 #if defined(_SC_NPROCESSORS_ONLN)
88 cpuCoreCount = sysconf(_SC_NPROCESSORS_ONLN);
89 #else
90 // _SC_NPROC_ONLN must be defined...
91 cpuCoreCount = sysconf(_SC_NPROC_ONLN);
92 #endif
93
94 CHECK_GE(cpuCoreCount, 1);
95 return cpuCoreCount;
96 }
97
VPXEncoder(int width,int height,int rateHz)98 FrameBufferSource::VPXEncoder::VPXEncoder(int width, int height, int rateHz)
99 : mWidth(width),
100 mHeight(height),
101 mRefreshRateHz(rateHz),
102 mCodecContext(nullptr, vpx_codec_destroy),
103 mForceIDRFrame(false),
104 mFirstFrame(true),
105 mStoredFrame(false),
106 mLastTimeUs(0) {
107
108 CHECK((width & 1) == 0);
109 CHECK((height & 1) == 0);
110 mSizeY = width * height;
111 mSizeUV = (width / 2) * (height / 2);
112 size_t totalSize = mSizeY + 2 * mSizeUV;
113 mI420Data = malloc(totalSize);
114
115 mCodecInterface = vpx_codec_vp8_cx();
116 mCodecConfiguration = std::make_unique<vpx_codec_enc_cfg_t>();
117
118 auto res = vpx_codec_enc_config_default(
119 mCodecInterface, mCodecConfiguration.get(), 0 /* usage */);
120
121 mCodecConfiguration->g_w = width;
122 mCodecConfiguration->g_h = height;
123 mCodecConfiguration->g_threads = std::min(GetCPUCoreCount(), 64);
124 mCodecConfiguration->g_error_resilient = false;
125 mCodecConfiguration->g_timebase.num = 1;
126 mCodecConfiguration->g_timebase.den = 1000000;
127 mCodecConfiguration->rc_target_bitrate = 2500; // This appears to match x264
128 mCodecConfiguration->rc_end_usage = VPX_VBR;
129 mCodecConfiguration->rc_dropframe_thresh = 0;
130 mCodecConfiguration->g_lag_in_frames = 0;
131
132 mCodecConfiguration->g_profile = 0;
133
134 CHECK_EQ(res, VPX_CODEC_OK);
135
136 mCodecContext.reset(new vpx_codec_ctx_t);
137
138 res = vpx_codec_enc_init(
139 mCodecContext.get(),
140 mCodecInterface,
141 mCodecConfiguration.get(),
142 0 /* flags */);
143
144 CHECK_EQ(res, VPX_CODEC_OK);
145
146 res = vpx_codec_control(mCodecContext.get(), VP8E_SET_TOKEN_PARTITIONS, 0);
147 CHECK_EQ(res, VPX_CODEC_OK);
148 }
149
~VPXEncoder()150 FrameBufferSource::VPXEncoder::~VPXEncoder() {
151 free(mI420Data);
152 mI420Data = nullptr;
153 }
154
forceIDRFrame()155 void FrameBufferSource::VPXEncoder::forceIDRFrame() {
156 mForceIDRFrame = true;
157 }
158
isForcingIDRFrame() const159 bool FrameBufferSource::VPXEncoder::isForcingIDRFrame() const {
160 return mForceIDRFrame;
161 }
162
storeFrame(const void * frame)163 void FrameBufferSource::VPXEncoder::storeFrame(const void *frame) {
164 uint8_t *yPlane = static_cast<uint8_t *>(mI420Data);
165 uint8_t *uPlane = yPlane + mSizeY;
166 uint8_t *vPlane = uPlane + mSizeUV;
167
168 libyuv::ABGRToI420(
169 static_cast<const uint8_t *>(frame),
170 mWidth * 4,
171 yPlane,
172 mWidth,
173 uPlane,
174 mWidth / 2,
175 vPlane,
176 mWidth / 2,
177 mWidth,
178 mHeight);
179 mStoredFrame = true;
180 }
181
encodeStoredFrame(int64_t timeUs)182 std::shared_ptr<SBuffer> FrameBufferSource::VPXEncoder::encodeStoredFrame(
183 int64_t timeUs) {
184 if (!mStoredFrame) {
185 return nullptr;
186 }
187 vpx_image_t raw_frame;
188 vpx_img_wrap(&raw_frame, VPX_IMG_FMT_I420, mWidth, mHeight,
189 2 /* stride_align */,
190 reinterpret_cast<unsigned char *>(mI420Data));
191
192 vpx_enc_frame_flags_t flags = 0;
193
194 if (mForceIDRFrame.exchange(false)) {
195 flags |= VPX_EFLAG_FORCE_KF;
196 }
197
198 uint32_t frameDuration;
199
200 if (!mFirstFrame) {
201 frameDuration = timeUs - mLastTimeUs;
202 } else {
203 frameDuration = 1000000 / mRefreshRateHz;
204 mFirstFrame = false;
205 }
206
207 mLastTimeUs = timeUs;
208
209 auto res = vpx_codec_encode(
210 mCodecContext.get(),
211 &raw_frame,
212 timeUs,
213 frameDuration,
214 flags,
215 VPX_DL_REALTIME);
216
217 if (res != VPX_CODEC_OK) {
218 LOG(ERROR) << "vpx_codec_encode failed w/ " << res;
219 return nullptr;
220 }
221
222 vpx_codec_iter_t iter = nullptr;
223 const vpx_codec_cx_pkt_t *packet;
224
225 std::shared_ptr<SBuffer> accessUnit;
226
227 while ((packet = vpx_codec_get_cx_data(mCodecContext.get(), &iter)) !=
228 nullptr) {
229 if (packet->kind == VPX_CODEC_CX_FRAME_PKT) {
230 LOG(VERBOSE)
231 << "vpx_codec_encode returned packet of size "
232 << packet->data.frame.sz;
233
234 if (accessUnit != nullptr) {
235 LOG(ERROR)
236 << "vpx_codec_encode returned more than one packet of "
237 "compressed data!";
238
239 return nullptr;
240 }
241
242 accessUnit.reset(new SBuffer(packet->data.frame.sz));
243
244 memcpy(accessUnit->data(),
245 packet->data.frame.buf,
246 packet->data.frame.sz);
247
248 accessUnit->time_us(timeUs);
249 } else {
250 LOG(INFO)
251 << "vpx_codec_encode returned a packet of type "
252 << packet->kind;
253 }
254 }
255
256 return accessUnit;
257 }
258
259 ////////////////////////////////////////////////////////////////////////////////
260
FrameBufferSource(Format format)261 FrameBufferSource::FrameBufferSource(Format format)
262 : mInitCheck(-ENODEV),
263 mState(STOPPED),
264 mFormat(format),
265 mScreenWidth(0),
266 mScreenHeight(0),
267 mScreenDpi(0),
268 mScreenRate(0),
269 mNumConsumers(0),
270 mOnFrameFn(nullptr) {
271 mInitCheck = 0;
272 }
273
~FrameBufferSource()274 FrameBufferSource::~FrameBufferSource() {
275 stop();
276 }
277
initCheck() const278 int32_t FrameBufferSource::initCheck() const {
279 return mInitCheck;
280 }
281
start()282 int32_t FrameBufferSource::start() {
283 std::lock_guard<std::mutex> autoLock(mLock);
284
285 if (mState != STOPPED) {
286 return 0;
287 }
288
289 switch (mFormat) {
290 case Format::VP8:
291 {
292 mEncoder.reset(
293 new VPXEncoder(mScreenWidth, mScreenHeight, mScreenRate));
294
295 break;
296 }
297
298 default:
299 LOG(FATAL) << "Should not be here.";
300 }
301
302 mState = RUNNING;
303
304 return 0;
305 }
306
stop()307 int32_t FrameBufferSource::stop() {
308 std::lock_guard<std::mutex> autoLock(mLock);
309
310 if (mState == STOPPED) {
311 return 0;
312 }
313
314 mState = STOPPING;
315
316 mState = STOPPED;
317
318 mEncoder.reset();
319
320 return 0;
321 }
322
pause()323 int32_t FrameBufferSource::pause() {
324 std::lock_guard<std::mutex> autoLock(mLock);
325
326 if (mState == PAUSED) {
327 return 0;
328 }
329
330 if (mState != RUNNING) {
331 return -EINVAL;
332 }
333
334 mState = PAUSED;
335
336 LOG(VERBOSE) << "Now paused.";
337
338 return 0;
339 }
340
resume()341 int32_t FrameBufferSource::resume() {
342 std::lock_guard<std::mutex> autoLock(mLock);
343
344 if (mState == RUNNING) {
345 return 0;
346 }
347
348 if (mState != PAUSED) {
349 return -EINVAL;
350 }
351
352 mState = RUNNING;
353
354 LOG(VERBOSE) << "Now running.";
355
356 return 0;
357 }
358
paused() const359 bool FrameBufferSource::paused() const {
360 return mState == PAUSED;
361 }
362
requestIDRFrame()363 int32_t FrameBufferSource::requestIDRFrame() {
364 mEncoder->forceIDRFrame();
365
366 return 0;
367 }
368
setScreenParams(const int32_t screenParams[4])369 void FrameBufferSource::setScreenParams(const int32_t screenParams[4]) {
370 mScreenWidth = screenParams[0];
371 mScreenHeight = screenParams[1];
372 mScreenDpi = screenParams[2];
373 mScreenRate = screenParams[3];
374 }
375
injectFrame(const void * data,size_t size)376 void FrameBufferSource::injectFrame(const void *data, size_t size) {
377 // Only used in the case of CrosVM operation.
378 (void)size;
379
380 std::lock_guard<std::mutex> autoLock(mLock);
381 if (mState != State::RUNNING) {
382 return;
383 }
384
385 mEncoder->storeFrame(data);
386 // Only encode and forward the frame when there are consumers connected
387 if (mNumConsumers) {
388 auto accessUnit = mEncoder->encodeStoredFrame(GetNowUs());
389 StreamingSource::onAccessUnit(accessUnit);
390 }
391 }
392
notifyNewStreamConsumer()393 void FrameBufferSource::notifyNewStreamConsumer() {
394 std::lock_guard<std::mutex> autoLock(mLock);
395 ++mNumConsumers;
396 if (mState != State::RUNNING) {
397 return;
398 }
399
400 mEncoder->forceIDRFrame();
401 auto accessUnit = mEncoder->encodeStoredFrame(GetNowUs());
402 if (!accessUnit) {
403 // nullptr means there isn't a stored frame to encode.
404 return;
405 }
406
407 StreamingSource::onAccessUnit(accessUnit);
408 }
409
notifyStreamConsumerDisconnected()410 void FrameBufferSource::notifyStreamConsumerDisconnected() {
411 std::lock_guard<std::mutex> autoLock(mLock);
412 --mNumConsumers;
413 }
414
415 } // namespace android
416