Fixing issue 1013. (simdjson#1016)

* Fixing issue 1013.

* Bumping to 0.4.6

Co-authored-by: Daniel Lemire <[email protected]>

Author: lemire

CMakeLists.txt

@@ -7,15 +7,12 @@ project(simdjson
 
 set(PROJECT_VERSION_MAJOR 0)
 set(PROJECT_VERSION_MINOR 4)
-set(PROJECT_VERSION_PATCH 5)
-set(SIMDJSON_LIB_VERSION "0.4.5" CACHE STRING "simdjson library version")
+set(PROJECT_VERSION_PATCH 6)
+set(SIMDJSON_SEMANTIC_VERSION "0.4.6" CACHE STRING "simdjson semantic version")
+set(SIMDJSON_LIB_VERSION "2.0.0" CACHE STRING "simdjson library version")
 set(SIMDJSON_LIB_SOVERSION "2" CACHE STRING "simdjson library soversion")
 set(SIMDJSON_GITHUB_REPOSITORY https://github.com/simdjson/simdjson)
 
-# The SIMDJSON_LIB_SOVERSION is 0 for versions before 0.3.0.
-# The SIMDJSON_LIB_SOVERSION is 1 for version 0.3.
-# The SIMDJSON_LIB_SOVERSION is 2 for version 0.4.
-
 include(GNUInstallDirs)
 include(cmake/simdjson-flags.cmake)
 include(cmake/simdjson-user-cmakecache.cmake)
@@ -47,8 +44,9 @@ add_subdirectory(windows)
 if(NOT(SIMDJSON_JUST_LIBRARY))
   add_subdirectory(dependencies) ## This needs to be before tools because of cxxopts
   add_subdirectory(tools)  ## This needs to be before tests because of cxxopts
+  add_subdirectory(singleheader)
 endif()
-add_subdirectory(singleheader)
+install(FILES singleheader/simdjson.h DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
 
 #
 # Compile tools / tests / benchmarks

Doxyfile

@@ -38,7 +38,7 @@ PROJECT_NAME           = simdjson
 # could be handy for archiving the generated documentation or if some version
 # control system is used.
 
-PROJECT_NUMBER         = "0.4.5"
+PROJECT_NUMBER         = "0.4.6"
 
 # Using the PROJECT_BRIEF tag one can provide an optional one line description
 # for a project that appears at the top of each page and should give viewer a

RELEASES.md

@@ -2,6 +2,23 @@
 
 ## Highlights
 
+- Test coverage has been greatly improved and we have resolved many static-analysis warnings on different systems.
+- We added a fast (8GB/s) minifier that works directly on JSON strings.
+- We added fast (10GB/s) UTF-8 validator that works directly on strings (any strings, including non-JSON).
+- The array and object elements have a constant-time size() method.
+- Performance improvements to the API (type(), get<>()).
+- The parse_many function (ndjson) has been entirely reworked. It now uses a single secondary thread instead of several new threads.
+- We have introduced a faster UTF-8 validation algorithm (lookup3) for all kernels (ARM, x64 SSE, x64 AVX).
+- C++11 support for older compilers and systems.
+- FreeBSD support (and tests).
+- We support the clang front-end compiler (clangcl) under Visual Studio.
+- It is now possible to target ARM platforms under Visual Studio.
+- The simdjson library will never abort or print to standard output/error.
+
+# 0.3
+
+## Highlights
+
 - **Multi-Document Parsing:** Read a bundle of JSON documents (ndjson) 2-4x faster than doing it
   individually. [API docs](https://github.com/simdjson/simdjson/blob/master/doc/basics.md#newline-delimited-json-ndjson-and-json-lines) / [Design Details](https://github.com/simdjson/simdjson/blob/master/doc/parse_many.md)
 - **Simplified API:** The API has been completely revamped for ease of use, including a new JSON

include/simdjson/simdjson_version.h

@@ -4,7 +4,7 @@
 #define SIMDJSON_SIMDJSON_VERSION_H
 
 /** The version of simdjson being used (major.minor.revision) */
-#define SIMDJSON_VERSION 0.4.5
+#define SIMDJSON_VERSION 0.4.6
 
 namespace simdjson {
 enum {
@@ -19,7 +19,7 @@ enum {
   /**
    * The revision (major.minor.REVISION) of simdjson being used.
    */
-  SIMDJSON_VERSION_REVISION = 5
+  SIMDJSON_VERSION_REVISION = 6
 };
 } // namespace simdjson
 

singleheader/amalgamate_demo.cpp

@@ -1,4 +1,4 @@
-/* auto-generated on Tue Jun 30 19:29:34 EDT 2020. Do not edit! */
+/* auto-generated on Wed Jul  1 14:00:57 EDT 2020. Do not edit! */
 
 #include <iostream>
 #include "simdjson.h"

singleheader/simdjson.cpp

@@ -1,4 +1,4 @@
-/* auto-generated on Tue Jun 30 19:29:34 EDT 2020. Do not edit! */
+/* auto-generated on Wed Jul  1 14:00:57 EDT 2020. Do not edit! */
 /* begin file src/simdjson.cpp */
 #include "simdjson.h"
 

singleheader/simdjson.h

@@ -1,4 +1,4 @@
-/* auto-generated on Tue Jun 30 19:29:34 EDT 2020. Do not edit! */
+/* auto-generated on Wed Jul  1 14:00:57 EDT 2020. Do not edit! */
 /* begin file include/simdjson.h */
 #ifndef SIMDJSON_H
 #define SIMDJSON_H
@@ -2040,7 +2040,7 @@ SIMDJSON_DISABLE_UNDESIRED_WARNINGS
 #define SIMDJSON_SIMDJSON_VERSION_H
 
 /** The version of simdjson being used (major.minor.revision) */
-#define SIMDJSON_VERSION 0.4.5
+#define SIMDJSON_VERSION 0.4.6
 
 namespace simdjson {
 enum {
@@ -2055,7 +2055,7 @@ enum {
   /**
    * The revision (major.minor.REVISION) of simdjson being used.
    */
-  SIMDJSON_VERSION_REVISION = 5
+  SIMDJSON_VERSION_REVISION = 6
 };
 } // namespace simdjson
 

src/CMakeLists.txt

@@ -84,7 +84,7 @@ if(NOT MSVC)
   # Please do not delete the following, our users want version numbers. See
   # https://github.com/simdjson/simdjson/issues/1014
   # https://github.com/simdjson/simdjson/issues/52
-  ###########
+  ########### 
   set_target_properties(simdjson PROPERTIES VERSION ${SIMDJSON_LIB_VERSION}	SOVERSION ${SIMDJSON_LIB_SOVERSION})
   ##########
   # End of the do-not-delete message.

tests/CMakeLists.txt

@@ -180,17 +180,17 @@ endif()
 if(NOT MSVC)
   ######
   # This tests is to guard us against ever again removing the soversion
-  # and version numbers from the library. See Bug
+  # number from the library. See Bug
   # https://github.com/simdjson/simdjson/issues/1014
   #####
-  get_target_property(REPORTED_SIMDJSON_VERSION simdjson VERSION)
   get_target_property(REPORTED_SIMDJSON_SOVERSION simdjson SOVERSION)
-  if(NOT ${REPORTED_SIMDJSON_VERSION} STREQUAL ${SIMDJSON_LIB_VERSION})
-    message(FATAL_ERROR "The library target does not have the proper version information." )
-  endif()
   if(NOT ${REPORTED_SIMDJSON_SOVERSION} STREQUAL ${SIMDJSON_LIB_SOVERSION})
     message(FATAL_ERROR "The library target does not have the proper soversion information." )
   endif()
+  get_target_property(REPORTED_SIMDJSON_VERSION simdjson VERSION)
+  if(NOT ${REPORTED_SIMDJSON_VERSION} STREQUAL ${SIMDJSON_LIB_VERSION})
+    message(FATAL_ERROR "The library target does not have the proper version information." )
+  endif()
 endif()
 
 add_subdirectory(compilation_failure_tests)

tools/release.py

@@ -7,7 +7,13 @@
 import subprocess
 import io
 import os
+import fileinput
+if sys.version_info < (3, 0):
+    sys.stdout.write("Sorry, requires Python 3.x or better\n")
+    sys.exit(1)
 
+def colored(r, g, b, text):
+    return "\033[38;2;{};{};{}m{} \033[38;2;255;255;255m".format(r, g, b, text)
 
 def extractnumbers(s):
     return tuple(map(int,re.findall("(\d+)\.(\d+)\.(\d+)",str(s))[0]))
@@ -22,17 +28,13 @@ def topaddedversionstring(major, minor, rev):
 branchresult = pipe.communicate()[0].decode().strip()
 
 if(branchresult != "master"):
-    print("release on master, you are on '"+branchresult+"'")
-    #sys.exit(-1)
-
+    print(colored(255, 0, 0, "We recommend that you release on master, you are on '"+branchresult+"'"))
 
 ret = subprocess.call(["git", "remote", "update"])
 
 if(ret != 0):
     sys.exit(ret)
 
-
-
 pipe = subprocess.Popen(["git", "log", "HEAD..", "--oneline"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
 uptodateresult = pipe.communicate()[0].decode().strip()
 
@@ -42,7 +44,7 @@ def topaddedversionstring(major, minor, rev):
 
 pipe = subprocess.Popen(["git", "rev-parse", "--show-toplevel"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
 maindir = pipe.communicate()[0].decode().strip()
-
+scriptlocation = os.path.dirname(os.path.abspath(__file__))
 
 print("repository: "+maindir)
 
@@ -79,7 +81,14 @@ def topaddedversionstring(major, minor, rev):
 atleastminor= (currentv[0] != newversion[0]) or (currentv[1] != newversion[1])
 
 if(atleastminor):
-    print("This is more than a revision.")
+    print(colored(0, 255, 0, "This is more than a revision."))
+    releasefile = maindir + os.sep + "RELEASES.md"
+    releasedata = open(releasefile).read()
+    pattern = re.compile("#\s+\d+\.\d+")
+    m = pattern.search(releasedata)
+    if(m == None):
+        print(colored(255, 0, 0, "You are preparing a new minor release and you have not yet updated RELEASES.md."))
+        sys.exit(-1)
 
 versionfilerel = os.sep + "include" + os.sep + "simdjson" + os.sep + "simdjson_version.h"
 versionfile = maindir + versionfilerel
@@ -114,10 +123,6 @@ def topaddedversionstring(major, minor, rev):
 
 print(versionfile + " modified")
 
-import fileinput
-import re
-
-
 newmajorversionstring = str(newversion[0])
 mewminorversionstring = str(newversion[1])
 newrevversionstring = str(newversion[2])
@@ -140,24 +145,53 @@ def topaddedversionstring(major, minor, rev):
     sonumber += 1
 
 for line in fileinput.input(cmakefile, inplace=1, backup='.bak'):
-    line = re.sub('SIMDJSON_LIB_VERSION "\d+\.\d+\.\d+','SIMDJSON_LIB_VERSION "'+newversionstring, line.rstrip())
-    line = re.sub('SIMDJSON_LIB_SOVERSION "\d+','SIMDJSON_LIB_SOVERSION "'+newmajorversionstring, line)
+    line = re.sub('SIMDJSON_SEMANTIC_VERSION "\d+\.\d+\.\d+','SIMDJSON_SEMANTIC_VERSION "'+newversionstring, line.rstrip())
+    line = re.sub('SIMDJSON_LIB_VERSION "\d+','SIMDJSON_LIB_VERSION "'+str(sonumber), line)
     line = re.sub('set\(PROJECT_VERSION_MAJOR \d+','set(PROJECT_VERSION_MAJOR '+newmajorversionstring, line)
     line = re.sub('set\(PROJECT_VERSION_MINOR \d+','set(PROJECT_VERSION_MINOR '+mewminorversionstring, line)
     line = re.sub('set\(PROJECT_VERSION_PATCH \d+','set(PROJECT_VERSION_PATCH '+newrevversionstring, line)
     line = re.sub('set\(SIMDJSON_LIB_SOVERSION \"\d+\"','set(SIMDJSON_LIB_SOVERSION \"'+str(sonumber)+'\"', line)
     print(line)
 
-
 print("modified "+cmakefile+", a backup was made")
+
+
 doxyfile = maindir + os.sep + "Doxyfile"
 for line in fileinput.input(doxyfile, inplace=1, backup='.bak'):
     line = re.sub('PROJECT_NUMBER         = "\d+\.\d+\.\d+','PROJECT_NUMBER         = "'+newversionstring, line.rstrip())
     print(line)
 print("modified "+doxyfile+", a backup was made")
 
-scriptlocation = os.path.dirname(os.path.abspath(__file__))
 
 
-print("Please run the tests before issuing a release, do make test && make amalgamate_test \n")
+cp = subprocess.run(["bash", "amalgamate.sh"], stdout=subprocess.DEVNULL, cwd=maindir+ os.sep + "singleheader")  # doesn't capture output
+if(cp.returncode != 0):
+    print("Failed to run amalgamate")
+
+cp = subprocess.run(["doxygen"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, cwd=maindir)  # doesn't capture output
+if(cp.returncode != 0):
+    print("Failed to run doxygen")
+
+#ipe = subprocess.Popen(["doxygen"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=maindir)
+#doxygenresult = pipe.communicate()[0].decode().strip()
+
+pattern = re.compile("https://simdjson.org/api/(\d+\.\d+\.\d+)/index.html")
+readmefile = maindir + os.sep + "README.md"
+readmedata = open(readmefile).read()
+m = pattern.search(readmedata)
+if m == None:
+    print(colored(255, 0, 0, 'I cannot find a link to the API documentation in your README?????'))
+else: 
+    detectedreadme = m.group(1)
+    print("found a link to your API documentation in the README file: "+detectedreadme+" ("+toversionstring(*newversion)+")")
+    if(atleastminor):
+       if(detectedreadme != toversionstring(*newversion)):
+           print(colored(255, 0, 0, "Consider updating the readme link to "+toversionstring(*newversion)))
+
+
+
+print("Please run the tests before issuing a release. \n")
 print("to issue release, enter \n git commit -a && git push  &&  git tag -a v"+toversionstring(*newversion)+" -m \"version "+toversionstring(*newversion)+"\" &&  git push --tags \n")
+
+
+