1 /* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
2 
3 Licensed under the Apache License, Version 2.0 (the "License");
4 you may not use this file except in compliance with the License.
5 You may obtain a copy of the License at
6 
7     http://www.apache.org/licenses/LICENSE-2.0
8 
9 Unless required by applicable law or agreed to in writing, software
10 distributed under the License is distributed on an "AS IS" BASIS,
11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 See the License for the specific language governing permissions and
13 limitations under the License.
14 ==============================================================================*/
15 #include <cstdarg>
16 #include <cstdio>
17 #include <cstdlib>
18 #include <fstream>
19 #include <map>
20 #include <sstream>
21 #include <gtest/gtest.h>
22 #include "re2/re2.h"
23 #include "tensorflow/lite/testing/parse_testdata.h"
24 #include "tensorflow/lite/testing/tflite_driver.h"
25 #include "tensorflow/lite/testing/util.h"
26 #include "tensorflow/core/lib/core/status_test_util.h"
27 #include "tensorflow/core/platform/env.h"
28 #include "tensorflow/core/platform/subprocess.h"
29 #include "tensorflow/core/util/command_line_flags.h"
30 
31 namespace tflite {
32 namespace testing {
33 
34 namespace {
35 bool FLAGS_ignore_known_bugs = true;
36 // As archive file names are test-specific, no default is possible.
37 //
38 // This test supports input as both zip and tar, as a stock android image does
39 // not have unzip but does have tar.
40 string* FLAGS_zip_file_path = new string;
41 string* FLAGS_tar_file_path = new string;
42 #ifndef __ANDROID__
43 string* FLAGS_unzip_binary_path = new string("/usr/bin/unzip");
44 string* FLAGS_tar_binary_path = new string("/bin/tar");
45 #else
46 string* FLAGS_unzip_binary_path = new string("/system/bin/unzip");
47 string* FLAGS_tar_binary_path = new string("/system/bin/tar");
48 #endif
49 bool FLAGS_use_nnapi = false;
50 bool FLAGS_ignore_unsupported_nnapi = false;
51 }  // namespace
52 
53 // TensorFlow system environment for file system called.
54 tensorflow::Env* env = tensorflow::Env::Default();
55 
56 // List of tests that are expected to fail when
57 //   --test_arg=--ignore_known_bugs=false
58 // Key is a substring of the test name and value is a bug number.
59 // TODO(ahentz): make sure we clean this list up frequently.
GetKnownBrokenTests()60 const std::map<string, string>& GetKnownBrokenTests() {
61   static const std::map<string, string>* const kBrokenTests =
62       new std::map<string, string>({
63 
64           // SpaceToBatchND only supports 4D tensors.
65           {R"(^\/space_to_batch_nd.*input_shape=\[1,4,4,4,1,1\])", "70848787"},
66 
67           // BatchToSpaceND only supports 4D tensors.
68           {R"(^\/batch_to_space_nd.*input_shape=\[8,2,2,2,1,1\])", "70848787"},
69 
70           // ResizeBilinear looks completely incompatible with Tensorflow
71           {R"(^\/resize_bilinear.*dtype=tf.int32)", "72401107"},
72 
73           // Select kernel doesn't support broadcasting yet.
74           {R"(^\/where.*1,2,3,1)", "134692786"},
75 
76           {R"(^\/div.*dtype=tf\.int64)", "119126484"},
77           {R"(^\/mul.*dtype=tf\.int64)", "119126484"},
78           {R"(^\/floor_div.*dtype=tf\.int64)", "119126484"},
79           {R"(^\/squared_difference.*dtype=tf\.int64)", "119126484"},
80       });
81   return *kBrokenTests;
82 }
83 
84 // Additional list of tests that are expected to fail when
85 //   --test_arg=--ignore_known_bugs=false
86 // and
87 //   --test_arg=--use_nnapi=true
88 // Note that issues related to lack of NNAPI support for a particular op are
89 // handled separately; this list is specifically for broken cases where
90 // execution produces broken output.
91 // Key is a substring of the test name and value is a bug number.
GetKnownBrokenNnapiTests()92 const std::map<string, string>& GetKnownBrokenNnapiTests() {
93   static const std::map<string, string>* const kBrokenNnapiTests =
94       new std::map<string, string>({
95           // Certain NNAPI kernels silently fail with int32 types.
96           {R"(^\/add.*dtype=tf\.int32)", "122987564"},
97           {R"(^\/concat.*dtype=tf\.int32)", "122987564"},
98           {R"(^\/mul.*dtype=tf\.int32)", "122987564"},
99           {R"(^\/space_to_depth.*dtype=tf\.int32)", "122987564"},
100 
101           // Certain NNAPI fully_connected shape permutations fail.
102           {R"(^\/fully_connected_constant_filter=True.*shape1=\[3,3\])",
103            "122987564"},
104           {R"(^\/fully_connected_constant_filter=True.*shape1=\[4,4\])",
105            "122987564"},
106           {R"(^\/fully_connected.*shape1=\[3,3\].*transpose_b=True)",
107            "122987564"},
108           {R"(^\/fully_connected.*shape1=\[4,4\].*shape2=\[4,1\])",
109            "122987564"},
110       });
111   return *kBrokenNnapiTests;
112 }
113 
114 // List of quantize tests that are probably to fail.
115 // Quantized tflite models has high diff error with tensorflow models.
116 // Key is a substring of the test name and value is a bug number.
117 // TODO(b/134594898): Remove these bugs and corresponding codes or move them to
118 // kBrokenTests after b/134594898 is fixed.
GetKnownQuantizeBrokenTests()119 const std::map<string, string>& GetKnownQuantizeBrokenTests() {
120   static const std::map<string, string>* const kQuantizeBrokenTests =
121       new std::map<string, string>({
122           {R"(^\/conv.*fully_quantize=True)", "134594898"},
123           {R"(^\/depthwiseconv.*fully_quantize=True)", "134594898"},
124           {R"(^\/sum.*fully_quantize=True)", "134594898"},
125           {R"(^\/l2norm.*fully_quantize=True)", "134594898"},
126           {R"(^\/prelu.*fully_quantize=True)", "156112683"},
127       });
128   return *kQuantizeBrokenTests;
129 }
130 
131 // Allows test data to be unarchived into a temporary directory and makes
132 // sure those temporary directories are removed later.
133 class ArchiveEnvironment : public ::testing::Environment {
134  public:
~ArchiveEnvironment()135   ~ArchiveEnvironment() override {}
136 
137   // Delete all temporary directories on teardown.
TearDown()138   void TearDown() override {
139     for (const auto& dir : temporary_directories_) {
140       tensorflow::int64 undeleted_dirs, undeleted_files;
141       TF_CHECK_OK(
142           env->DeleteRecursively(dir, &undeleted_dirs, &undeleted_files));
143     }
144     temporary_directories_.clear();
145   }
146 
147   // Unarchive `archive` file into a new temporary directory  `out_dir`.
UnArchive(const string & zip,const string & tar,string * out_dir)148   tensorflow::Status UnArchive(const string& zip, const string& tar,
149                                string* out_dir) {
150     string dir;
151     TF_CHECK_OK(MakeTemporaryDirectory(&dir));
152     tensorflow::SubProcess proc;
153     if (!zip.empty()) {
154       string unzip_binary = *FLAGS_unzip_binary_path;
155       TF_CHECK_OK(env->FileExists(unzip_binary));
156       TF_CHECK_OK(env->FileExists(zip));
157       proc.SetProgram(unzip_binary, {"unzip", "-d", dir, zip});
158     } else {
159       string tar_binary = *FLAGS_tar_binary_path;
160       TF_CHECK_OK(env->FileExists(tar_binary));
161       TF_CHECK_OK(env->FileExists(tar));
162       // 'o' needs to be explicitly set on Android so that
163       // untarring works as non-root (otherwise tries to chown
164       // files, which fails)
165       proc.SetProgram(tar_binary, {"tar", "xfo", tar, "-C", dir});
166     }
167     proc.SetChannelAction(tensorflow::CHAN_STDOUT, tensorflow::ACTION_PIPE);
168     proc.SetChannelAction(tensorflow::CHAN_STDERR, tensorflow::ACTION_PIPE);
169     if (!proc.Start())
170       return tensorflow::Status(tensorflow::error::UNKNOWN,
171                                 "unzip couldn't start");
172     string out, err;
173     int status = proc.Communicate(nullptr, &out, &err);
174     if (WEXITSTATUS(status) == 0) {
175       *out_dir = dir;
176       return tensorflow::Status::OK();
177     } else {
178       return tensorflow::Status(tensorflow::error::UNKNOWN,
179                                 "unzip failed. "
180                                 "stdout:\n" +
181                                     out + "\nstderr:\n" + err);
182     }
183   }
184 
185  private:
186   // Make a temporary directory and return its name in `temporary`.
MakeTemporaryDirectory(string * temporary)187   tensorflow::Status MakeTemporaryDirectory(string* temporary) {
188     if (env->LocalTempFilename(temporary)) {
189       TF_CHECK_OK(env->CreateDir(*temporary));
190       temporary_directories_.push_back(*temporary);
191       return tensorflow::Status::OK();
192     }
193     return tensorflow::Status(tensorflow::error::UNKNOWN,
194                               "make temporary directory failed");
195   }
196 
197   std::vector<string> temporary_directories_;
198 };
199 
200 // Return the singleton archive_environment.
archive_environment()201 ArchiveEnvironment* archive_environment() {
202   static ArchiveEnvironment* env = new ArchiveEnvironment;
203   return env;
204 }
205 
206 // Read the manifest.txt out of the unarchived archive file. Specifically
207 // `original_file` is the original zip file for error messages. `dir` is
208 // the temporary directory where the archive file has been unarchived and
209 // `test_paths` is the list of test prefixes that were in the manifest.
210 // Note, it is an error for a manifest to contain no tests.
ReadManifest(const string & original_file,const string & dir,std::vector<string> * test_paths)211 tensorflow::Status ReadManifest(const string& original_file, const string& dir,
212                                 std::vector<string>* test_paths) {
213   // Read the newline delimited list of entries in the manifest.
214   std::ifstream manifest_fp(dir + "/manifest.txt");
215   string manifest((std::istreambuf_iterator<char>(manifest_fp)),
216                   std::istreambuf_iterator<char>());
217   size_t pos = 0;
218   int added = 0;
219   while (true) {
220     size_t end_pos = manifest.find('\n', pos);
221     if (end_pos == string::npos) break;
222     string filename = manifest.substr(pos, end_pos - pos);
223     test_paths->push_back(dir + "/" + filename);
224     pos = end_pos + 1;
225     added += 1;
226   }
227   if (!added) {
228     string message = "Test had no examples: " + original_file;
229     return tensorflow::Status(tensorflow::error::UNKNOWN, message);
230   }
231   return tensorflow::Status::OK();
232 }
233 
234 // Get a list of tests from either zip or tar file
UnarchiveAndFindTestNames(const string & zip_file,const string & tar_file)235 std::vector<string> UnarchiveAndFindTestNames(const string& zip_file,
236                                               const string& tar_file) {
237   if (zip_file.empty() && tar_file.empty()) {
238     TF_CHECK_OK(tensorflow::Status(tensorflow::error::UNKNOWN,
239                                    "Neither zip_file nor tar_file was given"));
240   }
241   string decompress_tmp_dir;
242   TF_CHECK_OK(archive_environment()->UnArchive(zip_file, tar_file,
243                                                &decompress_tmp_dir));
244   std::vector<string> stuff;
245   if (!zip_file.empty()) {
246     TF_CHECK_OK(ReadManifest(zip_file, decompress_tmp_dir, &stuff));
247   } else {
248     TF_CHECK_OK(ReadManifest(tar_file, decompress_tmp_dir, &stuff));
249   }
250   return stuff;
251 }
252 
253 class OpsTest : public ::testing::TestWithParam<string> {};
254 
TEST_P(OpsTest,RunZipTests)255 TEST_P(OpsTest, RunZipTests) {
256   string test_path_and_label = GetParam();
257   string test_path = test_path_and_label;
258   string label = test_path_and_label;
259   size_t end_pos = test_path_and_label.find(' ');
260   if (end_pos != string::npos) {
261     test_path = test_path_and_label.substr(0, end_pos);
262     label = test_path_and_label.substr(end_pos + 1);
263   }
264   string tflite_test_case = test_path + "_tests.txt";
265   string tflite_dir = test_path.substr(0, test_path.find_last_of('/'));
266   string test_name = label.substr(label.find_last_of('/'));
267 
268   std::ifstream tflite_stream(tflite_test_case);
269   ASSERT_TRUE(tflite_stream.is_open()) << tflite_test_case;
270   tflite::testing::TfLiteDriver test_driver(
271       FLAGS_use_nnapi ? TfLiteDriver::DelegateType::kNnapi
272                       : TfLiteDriver::DelegateType::kNone);
273   bool fully_quantize = false;
274   if (label.find("fully_quantize=True") != std::string::npos) {
275     // TODO(b/134594898): Tighten this constraint.
276     test_driver.SetThreshold(0.2, 0.1);
277     fully_quantize = true;
278   }
279 
280   test_driver.SetModelBaseDir(tflite_dir);
281 
282   auto broken_tests = GetKnownBrokenTests();
283   if (FLAGS_use_nnapi) {
284     auto kBrokenNnapiTests = GetKnownBrokenNnapiTests();
285     broken_tests.insert(kBrokenNnapiTests.begin(), kBrokenNnapiTests.end());
286   }
287   auto quantize_broken_tests = GetKnownQuantizeBrokenTests();
288 
289   bool result = tflite::testing::ParseAndRunTests(&tflite_stream, &test_driver);
290   string message = test_driver.GetErrorMessage();
291 
292   if (!fully_quantize) {
293     string bug_number;
294     for (const auto& p : broken_tests) {
295       if (RE2::PartialMatch(test_name, p.first)) {
296         bug_number = p.second;
297         break;
298       }
299     }
300     if (bug_number.empty()) {
301       if (FLAGS_use_nnapi && FLAGS_ignore_unsupported_nnapi && !result) {
302         EXPECT_EQ(message, string("Failed to invoke interpreter")) << message;
303       } else {
304         EXPECT_TRUE(result) << message;
305       }
306     } else {
307       if (FLAGS_ignore_known_bugs) {
308         EXPECT_FALSE(result) << "Test was expected to fail but is now passing; "
309                                 "you can mark http://b/"
310                              << bug_number << " as fixed! Yay!";
311       } else {
312         EXPECT_TRUE(result)
313             << message << ": Possibly due to http://b/" << bug_number;
314       }
315     }
316   } else {
317     if (!result) {
318       string bug_number;
319       // See if the tests are potential quantize failures.
320       for (const auto& p : quantize_broken_tests) {
321         if (RE2::PartialMatch(test_name, p.first)) {
322           bug_number = p.second;
323           break;
324         }
325       }
326       EXPECT_FALSE(bug_number.empty());
327     }
328   }
329 }
330 
331 struct ZipPathParamName {
332   template <class ParamType>
operator ()tflite::testing::ZipPathParamName333   string operator()(const ::testing::TestParamInfo<ParamType>& info) const {
334     string param_name = info.param;
335     size_t last_slash = param_name.find_last_of("\\/");
336     if (last_slash != string::npos) {
337       param_name = param_name.substr(last_slash);
338     }
339     for (size_t index = 0; index < param_name.size(); ++index) {
340       if (!isalnum(param_name[index]) && param_name[index] != '_')
341         param_name[index] = '_';
342     }
343     return param_name;
344   }
345 };
346 
347 INSTANTIATE_TEST_CASE_P(tests, OpsTest,
348                         ::testing::ValuesIn(UnarchiveAndFindTestNames(
349                             *FLAGS_zip_file_path, *FLAGS_tar_file_path)),
350                         ZipPathParamName());
351 
352 }  // namespace testing
353 }  // namespace tflite
354 
main(int argc,char ** argv)355 int main(int argc, char** argv) {
356   ::testing::AddGlobalTestEnvironment(tflite::testing::archive_environment());
357 
358   std::vector<tensorflow::Flag> flags = {
359       tensorflow::Flag(
360           "ignore_known_bugs", &tflite::testing::FLAGS_ignore_known_bugs,
361           "If a particular model is affected by a known bug, the "
362           "corresponding test should expect the outputs to not match."),
363       tensorflow::Flag(
364           "tar_file_path", tflite::testing::FLAGS_tar_file_path,
365           "Required (or zip_file_path): Location of the test tar file."),
366       tensorflow::Flag(
367           "zip_file_path", tflite::testing::FLAGS_zip_file_path,
368           "Required (or tar_file_path): Location of the test zip file."),
369       tensorflow::Flag("unzip_binary_path",
370                        tflite::testing::FLAGS_unzip_binary_path,
371                        "Location of a suitable unzip binary."),
372       tensorflow::Flag("tar_binary_path",
373                        tflite::testing::FLAGS_tar_binary_path,
374                        "Location of a suitable tar binary."),
375       tensorflow::Flag("use_nnapi", &tflite::testing::FLAGS_use_nnapi,
376                        "Whether to enable the NNAPI delegate"),
377       tensorflow::Flag("ignore_unsupported_nnapi",
378                        &tflite::testing::FLAGS_ignore_unsupported_nnapi,
379                        "Don't fail tests just because delegation to NNAPI "
380                        "is not possible")};
381   bool success = tensorflow::Flags::Parse(&argc, argv, flags);
382   if (!success || (argc == 2 && !strcmp(argv[1], "--helpfull"))) {
383     fprintf(stderr, "%s", tensorflow::Flags::Usage(argv[0], flags).c_str());
384     return 1;
385   }
386 
387   ::tflite::LogToStderr();
388   // TODO(mikie): googletest arguments do not work - maybe the tensorflow flags
389   // parser removes them?
390   ::testing::InitGoogleTest(&argc, argv);
391   return RUN_ALL_TESTS();
392 }
393