1 /*
2  * Copyright (C) 2019 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "host/commands/assemble_cvd/image_aggregator.h"
18 
19 #include <sys/types.h>
20 #include <sys/stat.h>
21 #include <fcntl.h>
22 #include <stdio.h>
23 
24 #include <fstream>
25 #include <string>
26 #include <vector>
27 
28 #include <glog/logging.h>
29 #include <json/json.h>
30 #include <google/protobuf/text_format.h>
31 #include <sparse/sparse.h>
32 
33 #include "common/libs/fs/shared_buf.h"
34 #include "common/libs/fs/shared_fd.h"
35 #include "common/libs/utils/files.h"
36 #include "common/libs/utils/subprocess.h"
37 #include "host/commands/assemble_cvd/mbr.h"
38 #include "host/libs/config/cuttlefish_config.h"
39 #include "device/google/cuttlefish/host/commands/assemble_cvd/cdisk_spec.pb.h"
40 
41 namespace {
42 
43 const int GPT_HEADER_SIZE = 512 * 34;
44 const int GPT_FOOTER_SIZE = 512 * 33;
45 
46 const std::string BPTTOOL_FILE_PATH = "bin/cf_bpttool";
47 
BpttoolInput(const std::vector<ImagePartition> & partitions)48 Json::Value BpttoolInput(const std::vector<ImagePartition>& partitions) {
49   std::vector<off_t> file_sizes;
50   off_t total_size = 20 << 20; // 20 MB for padding
51   for (auto& partition : partitions) {
52     LOG(INFO) << "Examining " << partition.label;
53     auto file = cvd::SharedFD::Open(partition.image_file_path.c_str(), O_RDONLY);
54     if (!file->IsOpen()) {
55       LOG(FATAL) << "Could not open \"" << partition.image_file_path
56                  << "\": " << file->StrError();
57       break;
58     }
59     int fd = file->UNMANAGED_Dup();
60     auto sparse = sparse_file_import(fd, /* verbose */ false, /* crc */ false);
61     off_t partition_file_size = 0;
62     if (sparse) {
63       partition_file_size = sparse_file_len(sparse, /* sparse */ false,
64                                             /* crc */ true);
65       sparse_file_destroy(sparse);
66       close(fd);
67       LOG(INFO) << "was sparse";
68     } else {
69       partition_file_size = cvd::FileSize(partition.image_file_path);
70       if (partition_file_size == 0) {
71         LOG(FATAL) << "Could not get file size of \"" << partition.image_file_path
72                   << "\"";
73         break;
74       }
75       LOG(INFO) << "was not sparse";
76     }
77     LOG(INFO) << "size was " << partition_file_size;
78     total_size += partition_file_size;
79     file_sizes.push_back(partition_file_size);
80   }
81   Json::Value bpttool_input_json;
82   bpttool_input_json["settings"] = Json::Value();
83   bpttool_input_json["settings"]["disk_size"] = (Json::Int64) total_size;
84   bpttool_input_json["partitions"] = Json::Value(Json::arrayValue);
85   for (size_t i = 0; i < partitions.size(); i++) {
86     Json::Value partition_json;
87     partition_json["label"] = partitions[i].label;
88     partition_json["size"] = (Json::Int64) file_sizes[i];
89     partition_json["guid"] = "auto";
90     partition_json["type_guid"] = "linux_fs";
91     bpttool_input_json["partitions"].append(partition_json);
92   }
93   return bpttool_input_json;
94 }
95 
CreateFile(size_t len)96 std::string CreateFile(size_t len) {
97   char file_template[] = "/tmp/diskXXXXXX";
98   int fd = mkstemp(file_template);
99   if (fd < 0) {
100     LOG(FATAL) << "not able to create disk hole temp file";
101   }
102   char data[4096];
103   for (size_t i = 0; i < sizeof(data); i++) {
104     data[i] = '\0';
105   }
106   for (size_t i = 0; i < len + 2 * sizeof(data); i+= sizeof(data)) {
107     if (write(fd, data, sizeof(data)) < (ssize_t) sizeof(data)) {
108       LOG(FATAL) << "not able to write to disk hole temp file";
109     }
110   }
111   close(fd);
112   return std::string(file_template);
113 }
114 
MakeCompositeDiskSpec(const Json::Value & bpt_file,const std::vector<ImagePartition> & partitions,const std::string & header_file,const std::string & footer_file)115 CompositeDisk MakeCompositeDiskSpec(const Json::Value& bpt_file,
116                                     const std::vector<ImagePartition>& partitions,
117                                     const std::string& header_file,
118                                     const std::string& footer_file) {
119   CompositeDisk disk;
120   disk.set_version(1);
121   ComponentDisk* header = disk.add_component_disks();
122   header->set_file_path(header_file);
123   header->set_offset(0);
124   size_t previous_end = GPT_HEADER_SIZE;
125   for (auto& bpt_partition: bpt_file["partitions"]) {
126     if (bpt_partition["offset"].asUInt64() != previous_end) {
127       ComponentDisk* component = disk.add_component_disks();
128       component->set_file_path(CreateFile(bpt_partition["offset"].asUInt64() - previous_end));
129       component->set_offset(previous_end);
130     }
131     ComponentDisk* component = disk.add_component_disks();
132     for (auto& partition : partitions) {
133       if (bpt_partition["label"] == partition.label) {
134         component->set_file_path(partition.image_file_path);
135       }
136     }
137     component->set_offset(bpt_partition["offset"].asUInt64());
138     component->set_read_write_capability(ReadWriteCapability::READ_WRITE);
139     previous_end = bpt_partition["offset"].asUInt64() + bpt_partition["size"].asUInt64();
140   }
141   size_t footer_start = bpt_file["settings"]["disk_size"].asUInt64() - GPT_FOOTER_SIZE;
142   if (footer_start != previous_end) {
143     ComponentDisk* component = disk.add_component_disks();
144     component->set_file_path(CreateFile(footer_start - previous_end));
145     component->set_offset(previous_end);
146   }
147   ComponentDisk* footer = disk.add_component_disks();
148   footer->set_file_path(footer_file);
149   footer->set_offset(bpt_file["settings"]["disk_size"].asUInt64() - GPT_FOOTER_SIZE);
150   disk.set_length(bpt_file["settings"]["disk_size"].asUInt64());
151   return disk;
152 }
153 
JsonToFd(const Json::Value & json)154 cvd::SharedFD JsonToFd(const Json::Value& json) {
155   Json::FastWriter json_writer;
156   std::string json_string = json_writer.write(json);
157   cvd::SharedFD pipe[2];
158   cvd::SharedFD::Pipe(&pipe[0], &pipe[1]);
159   int written = pipe[1]->Write(json_string.c_str(), json_string.size());
160   if (written < 0) {
161     LOG(FATAL) << "Failed to write to pipe, errno is " << pipe[0]->GetErrno();
162   } else if (written < (int) json_string.size()) {
163     LOG(FATAL) << "Failed to write full json to pipe, only did " << written;
164   }
165   return pipe[0];
166 }
167 
FdToJson(cvd::SharedFD fd)168 Json::Value FdToJson(cvd::SharedFD fd) {
169   std::string contents;
170   cvd::ReadAll(fd, &contents);
171   Json::Reader reader;
172   Json::Value json;
173   if (!reader.parse(contents, json)) {
174     LOG(FATAL) << "Could not parse json: " << reader.getFormattedErrorMessages();
175   }
176   return json;
177 }
178 
BpttoolMakeTable(const cvd::SharedFD & input)179 cvd::SharedFD BpttoolMakeTable(const cvd::SharedFD& input) {
180   auto bpttool_path = vsoc::DefaultHostArtifactsPath(BPTTOOL_FILE_PATH);
181   cvd::Command bpttool_cmd(bpttool_path);
182   bpttool_cmd.AddParameter("make_table");
183   bpttool_cmd.AddParameter("--input=/dev/stdin");
184   bpttool_cmd.RedirectStdIO(cvd::Subprocess::StdIOChannel::kStdIn, input);
185   bpttool_cmd.AddParameter("--output_json=/dev/stdout");
186   cvd::SharedFD output_pipe[2];
187   cvd::SharedFD::Pipe(&output_pipe[0], &output_pipe[1]);
188   bpttool_cmd.RedirectStdIO(cvd::Subprocess::StdIOChannel::kStdOut, output_pipe[1]);
189   int success = bpttool_cmd.Start().Wait();
190   if (success != 0) {
191     LOG(FATAL) << "Unable to run bpttool. Exited with status " << success;
192   }
193   return output_pipe[0];
194 }
195 
BpttoolMakePartitionTable(cvd::SharedFD input)196 cvd::SharedFD BpttoolMakePartitionTable(cvd::SharedFD input) {
197   auto bpttool_path = vsoc::DefaultHostArtifactsPath(BPTTOOL_FILE_PATH);
198   cvd::Command bpttool_cmd(bpttool_path);
199   bpttool_cmd.AddParameter("make_table");
200   bpttool_cmd.AddParameter("--input=/dev/stdin");
201   bpttool_cmd.RedirectStdIO(cvd::Subprocess::StdIOChannel::kStdIn, input);
202   bpttool_cmd.AddParameter("--output_gpt=/dev/stdout");
203   cvd::SharedFD output_pipe[2];
204   cvd::SharedFD::Pipe(&output_pipe[0], &output_pipe[1]);
205   bpttool_cmd.RedirectStdIO(cvd::Subprocess::StdIOChannel::kStdOut, output_pipe[1]);
206   int success = bpttool_cmd.Start().Wait();
207   if (success != 0) {
208     LOG(FATAL) << "Unable to run bpttool. Exited with status " << success;
209   }
210   return output_pipe[0];
211 }
212 
CreateGptFiles(cvd::SharedFD gpt,const std::string & header_file,const std::string & footer_file)213 void CreateGptFiles(cvd::SharedFD gpt, const std::string& header_file,
214                     const std::string& footer_file) {
215   std::string content;
216   content.resize(GPT_HEADER_SIZE);
217   if (cvd::ReadExact(gpt, &content) < GPT_HEADER_SIZE) {
218     LOG(FATAL) << "Unable to run read full gpt. Errno is " << gpt->GetErrno();
219   }
220   auto header_fd = cvd::SharedFD::Open(header_file.c_str(), O_CREAT | O_RDWR, 0755);
221   if (cvd::WriteAll(header_fd, content) < GPT_HEADER_SIZE) {
222     LOG(FATAL) << "Unable to run write full gpt. Errno is " << gpt->GetErrno();
223   }
224   content.resize(GPT_FOOTER_SIZE);
225   if (cvd::ReadExact(gpt, &content) < GPT_FOOTER_SIZE) {
226     LOG(FATAL) << "Unable to run read full gpt. Errno is " << gpt->GetErrno();
227   }
228   auto footer_fd = cvd::SharedFD::Open(footer_file.c_str(), O_CREAT | O_RDWR, 0755);
229   if (cvd::WriteAll(footer_fd, content) < GPT_FOOTER_SIZE) {
230     LOG(FATAL) << "Unable to run write full gpt. Errno is " << gpt->GetErrno();
231   }
232 }
233 
BptToolMakeDiskImage(const std::vector<ImagePartition> & partitions,cvd::SharedFD table,const std::string & output)234 void BptToolMakeDiskImage(const std::vector<ImagePartition>& partitions,
235                           cvd::SharedFD table, const std::string& output) {
236   auto bpttool_path = vsoc::DefaultHostArtifactsPath(BPTTOOL_FILE_PATH);
237   cvd::Command bpttool_cmd(bpttool_path);
238   bpttool_cmd.AddParameter("make_disk_image");
239   bpttool_cmd.AddParameter("--input=/dev/stdin");
240   bpttool_cmd.AddParameter("--output=", cvd::AbsolutePath(output));
241   bpttool_cmd.RedirectStdIO(cvd::Subprocess::StdIOChannel::kStdIn, table);
242   for (auto& partition : partitions) {
243     auto abs_path = cvd::AbsolutePath(partition.image_file_path);
244     bpttool_cmd.AddParameter("--image=" + partition.label + ":" + abs_path);
245   }
246   int success = bpttool_cmd.Start().Wait();
247   if (success != 0) {
248     LOG(FATAL) << "Unable to run bpttool. Exited with status " << success;
249   }
250 }
251 
DeAndroidSparse(const std::vector<ImagePartition> & partitions)252 void DeAndroidSparse(const std::vector<ImagePartition>& partitions) {
253   for (const auto& partition : partitions) {
254     auto file = cvd::SharedFD::Open(partition.image_file_path.c_str(), O_RDONLY);
255     if (!file->IsOpen()) {
256       LOG(FATAL) << "Could not open \"" << partition.image_file_path
257                   << "\": " << file->StrError();
258       break;
259     }
260     int fd = file->UNMANAGED_Dup();
261     auto sparse = sparse_file_import(fd, /* verbose */ false, /* crc */ false);
262     if (!sparse) {
263       close(fd);
264       continue;
265     }
266     LOG(INFO) << "Desparsing " << partition.image_file_path;
267     std::string out_file_name = partition.image_file_path + ".desparse";
268     auto out_file = cvd::SharedFD::Open(out_file_name.c_str(), O_RDWR | O_CREAT | O_TRUNC,
269                                         S_IRUSR | S_IWUSR | S_IRGRP);
270     int write_fd = out_file->UNMANAGED_Dup();
271     int write_status = sparse_file_write(sparse, write_fd, /* gz */ false,
272                                          /* sparse */ false, /* crc */ false);
273     if (write_status < 0) {
274       LOG(FATAL) << "Failed to desparse \"" << partition.image_file_path << "\": " << write_status;
275     }
276     close(write_fd);
277     if (rename(out_file_name.c_str(), partition.image_file_path.c_str()) < 0) {
278       int error_num = errno;
279       LOG(FATAL) << "Could not move \"" << out_file_name << "\" to \""
280                  << partition.image_file_path << "\": " << strerror(error_num);
281     }
282     sparse_file_destroy(sparse);
283     close(fd);
284   }
285 }
286 
287 } // namespace
288 
AggregateImage(const std::vector<ImagePartition> & partitions,const std::string & output_path)289 void AggregateImage(const std::vector<ImagePartition>& partitions,
290                     const std::string& output_path) {
291   DeAndroidSparse(partitions);
292   auto bpttool_input_json = BpttoolInput(partitions);
293   auto input_json_fd = JsonToFd(bpttool_input_json);
294   auto table_fd = BpttoolMakeTable(input_json_fd);
295   BptToolMakeDiskImage(partitions, table_fd, output_path);
296 };
297 
CreateCompositeDisk(std::vector<ImagePartition> partitions,const std::string & header_file,const std::string & footer_file,const std::string & output_composite_path)298 void CreateCompositeDisk(std::vector<ImagePartition> partitions,
299                          const std::string& header_file,
300                          const std::string& footer_file,
301                          const std::string& output_composite_path) {
302   auto bpttool_input_json = BpttoolInput(partitions);
303   auto table_fd = BpttoolMakeTable(JsonToFd(bpttool_input_json));
304   auto table = FdToJson(table_fd);
305   auto partition_table_fd = BpttoolMakePartitionTable(JsonToFd(bpttool_input_json));
306   CreateGptFiles(partition_table_fd, header_file, footer_file);
307   auto composite_proto = MakeCompositeDiskSpec(table, partitions, header_file, footer_file);
308   std::ofstream composite(output_composite_path.c_str(), std::ios::binary | std::ios::trunc);
309   composite << "composite_disk\x1d";
310   composite_proto.SerializeToOstream(&composite);
311   composite.flush();
312 }
313 
CreateQcowOverlay(const std::string & crosvm_path,const std::string & backing_file,const std::string & output_overlay_path)314 void CreateQcowOverlay(const std::string& crosvm_path,
315                        const std::string& backing_file,
316                        const std::string& output_overlay_path) {
317   cvd::Command crosvm_qcow2_cmd(crosvm_path);
318   crosvm_qcow2_cmd.AddParameter("create_qcow2");
319   crosvm_qcow2_cmd.AddParameter("--backing_file=", backing_file);
320   crosvm_qcow2_cmd.AddParameter(output_overlay_path);
321   int success = crosvm_qcow2_cmd.Start().Wait();
322   if (success != 0) {
323     LOG(FATAL) << "Unable to run crosvm create_qcow2. Exited with status " << success;
324   }
325 }
326