File tree Expand file tree Collapse file tree 3 files changed +1
-10
lines changed Expand file tree Collapse file tree 3 files changed +1
-10
lines changed Original file line number Diff line number Diff line change 1- /*
2- * Copyright (c) Meta Platforms, Inc. and affiliates.
3- * All rights reserved.
4- *
5- * This source code is licensed under the BSD-style license found in the
6- * LICENSE file in the root directory of this source tree.
7- */
1+ // (c) Meta Platforms, Inc. and affiliates. Confidential and proprietary.
82
93// This file contains the implementation of create_regex with lookahead support
104
115#include < pytorch/tokenizers/pcre2_regex.h>
126#include < pytorch/tokenizers/regex.h>
137#include < pytorch/tokenizers/std_regex.h>
148
15- #include < iostream>
169#include < memory>
1710
1811namespace tokenizers {
Original file line number Diff line number Diff line change 2828#include < pytorch/tokenizers/base64.h>
2929#include < pytorch/tokenizers/tiktoken.h>
3030#include < algorithm>
31- #include < cinttypes>
3231#include < fstream>
3332#include < limits>
3433#include < unordered_set>
Original file line number Diff line number Diff line change 99#include < gtest/gtest.h>
1010
1111#include " pytorch/tokenizers/pcre2_regex.h"
12- #include " pytorch/tokenizers/re2_regex.h"
1312#include " pytorch/tokenizers/regex.h"
1413
1514using namespace tokenizers ;
You can’t perform that action at this time.
0 commit comments