Copybara configuration update.

PiperOrigin-RevId: 609931218

Author: dan-zheng

.github/workflows/build.yml

@@ -1,7 +1,7 @@
 name: Build
 
-# Trigger on push, pull request, or via manual dispatch.
-on: [push, pull_request, workflow_dispatch]
+# Trigger on push or via manual dispatch.
+on: [push, workflow_dispatch]
 
 jobs:
   build:

CMakeLists.txt

@@ -24,7 +24,7 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON)
 FetchContent_Declare(highway GIT_REPOSITORY https://github.com/google/highway.git GIT_TAG da250571a45826b21eebbddc1e50d0c1137dee5f)
 FetchContent_MakeAvailable(highway)
 
-## Note: absl needs to be installed by sentencepiece. This will only happen if
+## Note: absl meeds tp be installed by sentencepiece. This will only happen if
 ## cmake is invoked with -DSPM_ENABLE_SHARED=OFF and -DSPM_ABSL_PROVIDER=module
 FetchContent_Declare(sentencepiece GIT_REPOSITORY https://github.com/google/sentencepiece GIT_TAG 53de76561cfc149d3c01037f0595669ad32a5e7c)
 FetchContent_MakeAvailable(sentencepiece)
@@ -43,13 +43,14 @@ set(SOURCES
   util/args.h
   )
 
+add_compile_options($<$<CONFIG:Release>:-O2>)
 if(NOT CMAKE_BUILD_TYPE)
     set(CMAKE_BUILD_TYPE "Release")
 endif()
 
 # Allowable types for WEIGHT_TYPE:
 # float - slow, not recommended
-# hwy::bfloat16_t - bfloat16 as implemented by https://github.com/google/highway
+# hwy::bfloat16_t - bfloat16 as impemented by https://github.com/google/highway
 # SfpStream - 8-bit switched floating point (recommended)
 # NuqStream - experimental, work-in-progress
 option(WEIGHT_TYPE "Set weight type" "")
@@ -67,8 +68,6 @@ target_link_libraries(gemma hwy hwy_contrib sentencepiece)
 target_include_directories(gemma PRIVATE ./)
 FetchContent_GetProperties(sentencepiece)
 target_include_directories(gemma PRIVATE ${sentencepiece_SOURCE_DIR})
-target_compile_definitions(gemma PRIVATE $<$<PLATFORM_ID:Windows>:_CRT_SECURE_NO_WARNINGS NOMINMAX>)
-target_compile_options(gemma PRIVATE $<$<PLATFORM_ID:Windows>:-Wno-deprecated-declarations>)
 
 ## Library Target
 
@@ -78,5 +77,3 @@ set_target_properties(libgemma PROPERTIES PREFIX "")
 target_include_directories(libgemma PUBLIC ./)
 target_link_libraries(libgemma hwy hwy_contrib sentencepiece)
 target_include_directories(libgemma PRIVATE ${sentencepiece_SOURCE_DIR})
-target_compile_definitions(libgemma PRIVATE $<$<PLATFORM_ID:Windows>:_CRT_SECURE_NO_WARNINGS NOMINMAX>)
-target_compile_options(libgemma PRIVATE $<$<PLATFORM_ID:Windows>:-Wno-deprecated-declarations>)

CMakePresets.json

@@ -55,16 +55,6 @@ Before starting, you should have installed:
   least C++17.
 - `tar` for extracting archives from Kaggle.
 
-Building natively on Windows requires the Visual Studio 2012 Build Tools with the
-optional Clang/LLVM C++ frontend (`clang-cl`). This can be installed from the
-command line with
-[`winget`](https://learn.microsoft.com/en-us/windows/package-manager/winget/):
-
-```sh
-winget install --id Kitware.CMake
-winget install --id Microsoft.VisualStudio.2022.BuildTools --force --override "--passive --wait --add Microsoft.VisualStudio.Workload.VCTools;installRecommended --add Microsoft.VisualStudio.Component.VC.Llvm.Clang --add Microsoft.VisualStudio.Component.VC.Llvm.ClangToolset"
-```
-
 ### Step 1: Obtain model weights and tokenizer from Kaggle
 
 Visit [the Gemma model page on
@@ -117,7 +107,6 @@ runtime, create a build directory and generate the build files using `cmake`
 from the top-level project directory. For the 8-bit switched floating point
 weights (sfp), run cmake with no options:
 
-#### Unix-like Platforms
 ```sh
 cmake -B build
 ```
@@ -137,18 +126,17 @@ your weights, you can enter the `build/` directory and run `make` to build the
 `./gemma` executable:
 
 ```sh
-# Configure `build` directory
-cmake --preset make
-
-# Build project using make
-cmake --build --preset make -j [number of parallel threads to use]
+cd build
+make -j [number of parallel threads to use] gemma
 ```
 
 Replace `[number of parallel threads to use]` with a number - the number of
-cores available on your system is a reasonable heuristic.  For example,
-`make -j4 gemma` will build using 4 threads. If the `nproc` command is
-available, you can use `make -j$(nproc) gemma` as a reasonable default
-for the number of threads. 
+cores available on your system is a reasonable heuristic.
+
+For example, `make -j4 gemma` will build using 4 threads. If this is successful,
+you should now have a `gemma` executable in the `build/` directory. If the
+`nproc` command is available, you can use `make -j$(nproc) gemma` as a
+reasonable default for the number of threads. 
 
 If you aren't sure of the right value for the `-j` flag, you can simply run
 `make gemma` instead and it should still build the `./gemma` executable.
@@ -157,20 +145,6 @@ If you aren't sure of the right value for the `-j` flag, you can simply run
 > On Windows Subsystem for Linux (WSL) users should set the number of
 > parallel threads to 1. Using a larger number may result in errors.
 
-If the build is successful, you should now have a `gemma` executable in the `build/` directory.
-
-#### Windows
-
-```sh
-# Configure `build` directory
-cmake --preset windows
-
-# Build project using Visual Studio Build Tools
-cmake --build --preset windows -j [number of parallel threads to use]
-```
-
-If the build is successful, you should now have a `gemma.exe` executable in the `build/` directory.
-
 ### Step 4: Run
 
 You can now run `gemma` from inside the `build/` directory.

README.md

@@ -16,16 +16,11 @@
 // copybara:import_next_line:gemma_cpp
 #include "compression/blob_store.h"
 
+#include <fcntl.h>  // open
 #include <stdint.h>
 #include <stdio.h>     // SEEK_END - unistd isn't enough for IDE.
 #include <sys/stat.h>  // O_RDONLY
-#include <fcntl.h>  // open
-#if HWY_OS_WIN
-#include <io.h>  // read, write, close
-#include <fileapi.h>
-#else
-#include <unistd.h>    // read, write, close
-#endif
+#include <unistd.h>    // read, close
 
 #include <atomic>
 #include <vector>
@@ -35,54 +30,6 @@
 #include "hwy/contrib/thread_pool/thread_pool.h"
 #include "hwy/detect_compiler_arch.h"
 
-namespace {
-#if HWY_OS_WIN
-
-// pread is not supported on Windows
-static int64_t pread(int fd, void* buf, uint64_t size, uint64_t offset) {
-  HANDLE file = reinterpret_cast<HANDLE>(_get_osfhandle(fd));
-  if (file == INVALID_HANDLE_VALUE) {
-    return -1;
-  }
-
-  OVERLAPPED overlapped = {0};
-  overlapped.Offset = offset & 0xFFFFFFFF;
-  overlapped.OffsetHigh = (offset >> 32) & 0xFFFFFFFF;
-
-  DWORD bytes_read;
-  if (!ReadFile(file, buf, size, &bytes_read, &overlapped)) {
-    if (GetLastError() != ERROR_HANDLE_EOF) {
-      return -1;
-    }
-  }
-
-  return bytes_read;
-}
-
-// pwrite is not supported on Windows
-static int64_t pwrite(int fd, const void* buf, uint64_t size, uint64_t offset) {
-  HANDLE file = reinterpret_cast<HANDLE>(_get_osfhandle(fd));
-  if (file == INVALID_HANDLE_VALUE) {
-    return -1;
-  }
-
-  OVERLAPPED overlapped = {0};
-  overlapped.Offset = offset & 0xFFFFFFFF;
-  overlapped.OffsetHigh = (offset >> 32) & 0xFFFFFFFF;
-
-  DWORD bytes_written;
-  if (!WriteFile(file, buf, size, &bytes_written, &overlapped)) {
-    if (GetLastError() != ERROR_HANDLE_EOF) {
-      return -1;
-    }
-  }
-
-  return bytes_written;
-}
-
-#endif
-}
-
 namespace gcpp {
 
 hwy::uint128_t MakeKey(const char* string) {
@@ -117,30 +64,19 @@ static void EnqueueChunkRequests(uint64_t offset, uint64_t size, uint8_t* data,
   }
 }
 
-
 struct IO {
   // Returns size in bytes or 0.
   static uint64_t FileSize(const char* filename) {
     int fd = open(filename, O_RDONLY);
-    if (fd < 0) {
-      return 0;
+    if (fd >= 0) {
+      const off_t size = lseek(fd, 0, SEEK_END);
+      HWY_ASSERT(close(fd) != -1);
+      if (size != static_cast<off_t>(-1)) {
+        return static_cast<uint64_t>(size);
+      }
     }
 
-#if HWY_OS_WIN
-    const int64_t size = _lseeki64(fd, 0, SEEK_END);
-    HWY_ASSERT(close(fd) != -1);
-    if (size < 0) {
-      return 0;
-    }
-#else
-    const off_t size = lseek(fd, 0, SEEK_END);
-    HWY_ASSERT(close(fd) != -1);
-    if (size == static_cast<off_t>(-1)) {
-      return 0;
-    }
-#endif
-
-    return static_cast<uint64_t>(size);
+    return 0;
   }
 
   static bool Read(int fd, uint64_t offset, uint64_t size, void* to) {
@@ -316,14 +252,7 @@ class BlobStore {
 #pragma pack(pop)
 
 BlobError BlobReader::Open(const char* filename) {
-#if HWY_OS_WIN
-  DWORD flags = FILE_ATTRIBUTE_NORMAL | FILE_FLAG_SEQUENTIAL_SCAN;
-  HANDLE file = CreateFileA(filename, GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, flags, nullptr);
-  if (file == INVALID_HANDLE_VALUE) return __LINE__;
-  fd_ = _open_osfhandle(reinterpret_cast<intptr_t>(file), _O_RDONLY);
-#else
   fd_ = open(filename, O_RDONLY);
-#endif
   if (fd_ < 0) return __LINE__;
 
 #if _POSIX_C_SOURCE >= 200112L
@@ -401,14 +330,7 @@ BlobError BlobWriter::WriteAll(hwy::ThreadPool& pool,
       keys_.data(), blobs_.data(), keys_.size());
 
   // Create/replace existing file.
-#if HWY_OS_WIN
-  DWORD flags = FILE_ATTRIBUTE_NORMAL;
-  HANDLE file = CreateFileA(filename, GENERIC_WRITE, 0, nullptr, CREATE_ALWAYS, flags, nullptr);
-  if (file == INVALID_HANDLE_VALUE) return __LINE__;
-  const int fd = _open_osfhandle(reinterpret_cast<intptr_t>(file), _O_WRONLY);
-#else
   const int fd = open(filename, O_CREAT | O_RDWR | O_TRUNC, 0644);
-#endif
   if (fd < 0) return __LINE__;
 
   std::atomic_flag err = ATOMIC_FLAG_INIT;
@@ -419,7 +341,6 @@ BlobError BlobWriter::WriteAll(hwy::ThreadPool& pool,
                err.test_and_set();
              }
            });
-  HWY_ASSERT(close(fd) != -1);
   if (err.test_and_set()) return __LINE__;
   return 0;
 }

compression/blob_store.cc

@@ -60,8 +60,7 @@
 #include "hwy/aligned_allocator.h"
 #include "hwy/base.h"
 #include "hwy/contrib/thread_pool/thread_pool.h"
-// copybara:import_next_line:sentencepiece
-#include "src/sentencepiece_processor.h"
+#include "sentencepiece_processor.h"
 
 namespace gcpp {
 

gemma.cc

@@ -33,8 +33,7 @@
 #include "hwy/aligned_allocator.h"
 #include "hwy/base.h"  // hwy::bfloat16_t
 #include "hwy/contrib/thread_pool/thread_pool.h"
-// copybara:import_next_line:sentencepiece
-#include "src/sentencepiece_processor.h"
+#include "sentencepiece_processor.h"
 
 namespace gcpp {
 

gemma.h

@@ -18,9 +18,7 @@
 #ifndef THIRD_PARTY_GEMMA_CPP_UTIL_APP_H_
 #define THIRD_PARTY_GEMMA_CPP_UTIL_APP_H_
 
-#if HWY_OS_LINUX
 #include <sched.h>
-#endif
 #include <stddef.h>
 
 #include <algorithm>  // std::clamp