Skip to content

Commit

Permalink
Reverted back to PythonCall in hopes for it to work all the time and not
Browse files Browse the repository at this point in the history
just sporadically.
  • Loading branch information
yakir12 committed Dec 23, 2024
1 parent 6e4af36 commit 5c0db73
Show file tree
Hide file tree
Showing 9 changed files with 156 additions and 2,389 deletions.
5 changes: 5 additions & 0 deletions CondaPkg.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
channels = ["conda-forge"]

[deps]
opencv = ""
numpy = ""
13 changes: 8 additions & 5 deletions Project.toml
Original file line number Diff line number Diff line change
@@ -1,19 +1,20 @@
name = "CameraCalibrations"
uuid = "70c3cbac-2c23-4b2d-966c-5a25c0a540fb"
authors = ["Yakir <[email protected]> and contributors"]
version = "0.7.3"
version = "0.6.0"

[deps]
CondaPkg = "992eb4ea-22a4-4c89-a5bb-47a3300528ab"
CoordinateTransformations = "150eb455-5306-5404-9cee-2592286d6298"
FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549"
ImageBase = "c817782e-172a-44cc-b673-b171935fbb9e"
ImageDraw = "4381153b-2b60-58ae-a1ba-fd683676385f"
ImageBase = "c817782e-172a-44cc-b673-b171935fbb9e"
ImageIO = "82e4d734-157c-48bb-816b-45c225c6df19"
ImageTransformations = "02fcd773-0e25-5acc-982a-7f6622650795"
JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1"
LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
OpenCV = "f878e3a2-a245-4720-8660-60795d644f2a"
Polynomials = "f27b6e38-b328-58d1-80ce-0feddd5e7a45"
PythonCall = "6099a3de-0909-46bc-b1f4-468b9a2dfc0d"
Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
Rotations = "6038ab10-8711-5258-84ad-4b1120ba62dc"
StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
Expand All @@ -23,6 +24,7 @@ StructTypes = "856f2bd8-1eba-4b0a-8007-ebc267875bd4"
[compat]
Aqua = "0.8"
ColorTypes = "0.12.0"
CondaPkg = "0.2"
CoordinateTransformations = "0.6"
FileIO = "1"
ImageBase = "0.1.7"
Expand All @@ -31,9 +33,9 @@ ImageIO = "0.6"
ImageTransformations = "0.9, 0.10"
JSON3 = "1.14.1"
LinearAlgebra = "1"
OpenCV = "4.6.1"
PaddedViews = "0.5.12"
Polynomials = "3, 4"
PythonCall = "0.9"
Random = "1.10"
Rotations = "1"
StaticArrays = "1"
Expand All @@ -46,11 +48,12 @@ julia = "1.10"
Aqua = "4c88cf16-eb10-579e-8560-4a9242c79595"
ColorTypes = "3da002f7-5984-5a60-b8a6-cbb66c0b333f"
FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549"
ImageTransformations = "02fcd773-0e25-5acc-982a-7f6622650795"
JSON3 = "0f8b85d8-7281-11e9-16c2-39a750bddbf1"
LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
PaddedViews = "5432bcbf-9aad-5242-b902-cca2824c8663"
StaticArrays = "90137ffa-7385-5640-81b9-e52037218182"
Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

[targets]
test = ["Aqua", "ColorTypes", "FileIO", "JSON3", "LinearAlgebra", "PaddedViews", "StaticArrays", "Test"]
test = ["Aqua", "ColorTypes", "FileIO", "ImageTransformations", "JSON3", "LinearAlgebra", "PaddedViews", "StaticArrays", "Test"]
15 changes: 12 additions & 3 deletions src/CameraCalibrations.jl
Original file line number Diff line number Diff line change
Expand Up @@ -2,19 +2,28 @@ module CameraCalibrations

using LinearAlgebra, Statistics, Random
import ImageIO, FileIO
using OpenCV, ImageBase, StaticArrays, ImageDraw, ImageTransformations
using PythonCall, ImageBase, StaticArrays, ImageDraw, ImageTransformations
using Rotations, CoordinateTransformations, Polynomials

using JSON3, StructTypes

export fit, Calibration, CalibrationIO, rectification, RowCol, XYZ

include("meta.jl")
include("io.jl")
include("detect_fit.jl")
include("buildcalibrations.jl")
include("plot_calibration.jl")

const CRITERIA = OpenCV.TermCriteria(OpenCV.TERM_CRITERIA_EPS + OpenCV.TERM_CRITERIA_MAX_ITER, 30, 0.001)
# using CondaPkg
# CondaPkg.add.(["numpy", "opencv"])

const cv2 = PythonCall.pynew()
const np = PythonCall.pynew()

export fit, Calibration, RowCol, XYZ, rectification
function __init__()
PythonCall.pycopy!(cv2, pyimport("cv2"))
PythonCall.pycopy!(np, pyimport("numpy"))
end

end
99 changes: 57 additions & 42 deletions src/detect_fit.jl
Original file line number Diff line number Diff line change
@@ -1,57 +1,69 @@
# the following convertion functions are necessary due to some overly eager type checking in the python functions.
convert_from_py_corners(py_corners, n_corners) = reshape(RowCol.(eachslice(PyArray(py_corners); dims = 1)), n_corners)

convert_to_py_objpoints(objpoints, n) = PyList(fill(np.array(Float32.(reshape(reduce((x1, x2) -> hcat(x1, Vector(x2)), objpoints)', (1, length(objpoints), 3)))), n))

convert_to_py_imgpointss(imgpointss) = PyList([np.array(reshape(reduce((x1, x2) -> hcat(x1, Vector(x2)), imgpoints)', 1, length(imgpoints), 2)) for imgpoints in imgpointss])

"""
get_object_points
Produce the real-world locations of the corners of the checkerboard.
"""
function get_object_points(n_corners)
objpoints = Matrix{XYZ}(undef, n_corners)
for i in CartesianIndices(n_corners)
x, y = Tuple(i) .- 1
objpoints[i] = XYZ(x, y, 0)
end
return objpoints
end

"""
_detect_corners
Wraps OpenCV function to auto-detect corners in an image.
"""
function _detect_corners(file, n_corners, sz)
img = FileIO.load(file)
gry = reshape(rawview(channelview(Gray.(img))), 1, sz...)
cv_n_corners = OpenCV.Size{Int32}(n_corners...)
_cv_corners = OpenCV.Mat(Array{Float32}(undef, 2, 1, prod(n_corners)))
ret, cv_corners = OpenCV.findChessboardCorners(gry, cv_n_corners, _cv_corners,
OpenCV.CALIB_CB_ADAPTIVE_THRESH +
OpenCV.CALIB_CB_FAST_CHECK +
OpenCV.CALIB_CB_EXHAUSTIVE + # Run an exhaustive search to improve detection rate.
OpenCV.CALIB_CB_ACCURACY # Up sample input image to improve sub-pixel accuracy due to aliasing effects.
)
!ret && return missing
# corners = reshape(RowCol.(eachslice(cv_corners, dims = 3)), n_corners)
ref_corners = OpenCV.cornerSubPix(gry, cv_corners, OpenCV.Size{Int32}(11,11), OpenCV.Size{Int32}(-1,-1), CRITERIA)
corners = reshape(RowCol.(eachslice(ref_corners, dims = 3)), n_corners)
return (file, corners)
function _detect_corners(file, n_corners)
criteria = (cv2.TERM_CRITERIA_EPS + cv2.TERM_CRITERIA_MAX_ITER, 30, 0.001)

img = cv2.imread(file)
gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
ret, py_corners = cv2.findChessboardCorners(gray, n_corners, cv2.CALIB_CB_ADAPTIVE_THRESH + cv2.CALIB_CB_NORMALIZE_IMAGE + cv2.CALIB_CB_FAST_CHECK)

!Bool(ret) && return missing

ref_corners = cv2.cornerSubPix(gray, py_corners, (11,11),(-1,-1), criteria)
flp_corners = np.flip(ref_corners, axis = 2)
corners = convert_from_py_corners(flp_corners, n_corners)
return (file, reverse(corners, dims = 1))
end

function detect_corners(_files, n_corners)
fi = skipmissing(_detect_corners.(_files, Ref(n_corners)))
return (; files = first.(fi), imgpointss = last.(fi))
end

"""
fit_model
Wraps OpenCV function to fit a camera model to given object and image points.
"""
function fit_model(sz, objpoints, imgpointss, n_corners, with_distortion, aspect)
nfiles = length(imgpointss)

objectPoints = OpenCV.InputArray[Float32.(reshape(stack(objpoints), 3, 1, :)) for _ in 1:nfiles]
imagePoints = OpenCV.InputArray[Float32.(reshape(stack(imgpoints), 2, 1, :)) for imgpoints in imgpointss]
imageSize = OpenCV.Size{Int32}(sz...)
flags = cv2.CALIB_ZERO_TANGENT_DIST + cv2.CALIB_FIX_K3 + cv2.CALIB_FIX_K2 + (with_distortion ? 0 : cv2.CALIB_FIX_K1) + cv2.CALIB_FIX_ASPECT_RATIO

cammat = convert(Matrix{Float32}, I(3))
cammat[1,:] .= aspect
cameraMatrix = OpenCV.Mat(Float32.(reshape(cammat, 1, 3, 3)))
# distCoeffs = OpenCV.Mat(Array{Float32}(undef, 1, 1, 5))
distCoeffs = OpenCV.Mat(zeros(Float32, 1, 1, 5))
cammatrix = convert(Matrix{Float32}, I(3))
cammatrix[1] = aspect

flags = OpenCV.CALIB_ZERO_TANGENT_DIST + OpenCV.CALIB_FIX_K3 + OpenCV.CALIB_FIX_K2 + (with_distortion ? 0 : OpenCV.CALIB_FIX_K1) + OpenCV.CALIB_FIX_ASPECT_RATIO
cammatrix = np.ascontiguousarray(np.array(cammatrix)) # https://stackoverflow.com/a/50128836/2261957

# r = OpenCV.InputArray[Array{Float32}(undef, 3, 1, prod(n_corners)) for _ in 1:nfiles]
r = OpenCV.InputArray[zeros(Float32, 3, 1, prod(n_corners)) for _ in 1:nfiles]
# t = OpenCV.InputArray[Array{Float32}(undef, 3, 1, prod(n_corners)) for _ in 1:nfiles]
t = OpenCV.InputArray[zeros(Float32, 3, 1, prod(n_corners)) for _ in 1:nfiles]
_, py_mtx, py_dist, py_rvecs, py_tvecs = cv2.calibrateCamera(convert_to_py_objpoints(objpoints, length(imgpointss)), convert_to_py_imgpointss(imgpointss), np.flip(sz), cammatrix, nothing; flags)

x, mtx, dist, rvecs, tvecs = OpenCV.calibrateCamera(objectPoints, imagePoints, imageSize, cameraMatrix, distCoeffs, r, t, flags, CRITERIA)
k, _ = PyArray(py_dist)
@assert with_distortion || k == 0 "distortion was $with_distortion but k isn't zero:" k

k, _ = dist
Rs = [vec(pyconvert(Matrix{Float64}, x)) for x in py_rvecs]

Rs = vec.(rvecs)
ts = vec.(tvecs)
ts = [vec(pyconvert(Matrix{Float64}, x)) for x in py_tvecs]

mtx = reshape(mtx, 3, 3)'
mtx = Matrix(PyArray(py_mtx))
frow = mtx[1,1]
fcol = mtx[2,2]
crow = mtx[1,3]
Expand All @@ -61,12 +73,15 @@ function fit_model(sz, objpoints, imgpointss, n_corners, with_distortion, aspec
end

function detect_fit(_files, n_corners, with_distortion, aspect)
sz = size(FileIO.load(first(_files)))
fi = skipmissing(_detect_corners.(_files, Ref(n_corners), Ref(sz)))
@assert !isempty(fi) "No checkers were detected in any of the images, perhaps try a different `n_corners`."
files = first.(fi)
imgpointss = last.(fi)
objpoints = XYZ.(Tuple.(CartesianIndices((0:(n_corners[1] - 1), 0:(n_corners[2] - 1), 0:0))))
files, imgpointss = detect_corners(_files, n_corners)

objpoints = get_object_points(n_corners)
sz = size(FileIO.load(files[1]))

k, Rs, ts, frow, fcol, crow, ccol = fit_model(sz, objpoints, imgpointss, n_corners, with_distortion, aspect)

return (; files, objpoints, imgpointss, sz, k, Rs, ts, frow, fcol, crow, ccol)
end



2 changes: 1 addition & 1 deletion test/example/corners.json
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"2.png":[[99,119],[143,121],[186,122],[229,124],[270,126],[99,163],[143,165],[186,166],[228,168],[269,169],[99,207],[143,208],[185,209],[227,210],[268,211],[99,250],[143,251],[185,251],[226,252],[267,252],[98,293],[142,293],[184,293],[225,293],[266,293],[98,336],[141,335],[183,334],[224,334],[265,334],[97,378],[140,377],[183,376],[224,376],[264,375],[97,420],[140,419],[182,418],[223,416],[263,415]],"6.png":[[112,177],[136,179],[159,180],[181,181],[202,182],[112,201],[136,202],[159,203],[181,204],[203,205],[113,225],[137,226],[160,226],[182,227],[203,227],[113,249],[137,249],[160,249],[182,249],[203,249],[113,273],[137,273],[160,272],[182,272],[204,271],[114,297],[138,296],[161,295],[183,294],[204,293],[114,322],[138,320],[161,319],[183,317],[205,316],[114,346],[138,344],[161,342],[184,340],[205,339]],"3.png":[[85,161],[119,163],[152,164],[183,166],[213,167],[86,179],[122,181],[157,182],[189,183],[221,184],[87,200],[126,201],[162,202],[197,203],[230,203],[88,223],[129,224],[168,224],[205,224],[240,224],[89,250],[134,250],[175,250],[214,249],[251,249],[91,282],[139,280],[183,279],[225,278],[265,276],[92,320],[144,316],[192,314],[237,311],[280,309],[93,365],[151,360],[203,355],[253,351],[299,348]],"1.png":[[86,197],[141,195],[194,194],[246,192],[298,190],[93,243],[144,241],[193,239],[241,236],[290,234],[99,283],[146,280],[192,277],[238,275],[283,273],[104,317],[148,314],[192,311],[234,309],[277,307],[108,348],[150,345],[191,342],[231,340],[271,338],[112,376],[152,373],[191,370],[229,367],[267,365],[115,400],[153,397],[190,394],[226,392],[262,389],[119,422],[154,419],[190,416],[224,414],[258,411]],"5.png":[[61,109],[93,101],[128,92],[167,83],[212,72],[63,151],[95,146],[130,140],[169,133],[213,126],[66,193],[97,190],[133,187],[171,184],[215,179],[68,235],[100,234],[135,233],[173,232],[216,231],[71,275],[102,276],[137,278],[175,279],[218,281],[73,315],[104,318],[139,322],[177,326],[220,330],[75,355],[106,360],[141,366],[179,372],[221,380],[78,394],[108,401],[143,409],[181,418],[223,428]],"4.png":[[106,90],[141,102],[171,113],[198,122],[221,130],[106,135],[142,144],[172,152],[198,159],[221,165],[107,181],[143,187],[173,192],[199,196],[222,200],[108,227],[144,229],[174,231],[200,232],[222,234],[109,272],[144,271],[174,270],[200,269],[223,268],[109,318],[145,313],[175,309],[201,305],[224,302],[109,365],[145,356],[175,348],[202,342],[225,336],[110,411],[146,399],[176,389],[202,379],[225,372]]}
{"2.png":[[263.18216,415.31693],[222.93118,416.49454],[181.82751,417.67328],[140.01404,418.76404],[97.37207,419.78665],[264.3202,375.35947],[223.69255,375.78485],[182.53548,376.49304],[140.428,377.42496],[97.398506,378.3202],[265.06757,334.38934],[224.35864,334.32297],[183.21829,334.4877],[141.21309,335.06955],[97.73969,335.8214],[265.71442,293.3704],[224.86827,292.9574],[183.86363,292.75208],[141.94615,292.80072],[98.49167,293.08423],[266.52853,252.29192],[225.67264,251.7298],[184.54213,251.30998],[142.50674,250.73573],[98.81908,250.27014],[267.80743,210.75446],[226.5611,210.18546],[185.24628,209.39569],[142.76242,208.31534],[98.848656,206.93794],[269.19818,168.52621],[227.73825,167.5486],[185.80865,166.38515],[142.85547,164.67586],[98.78267,162.90886],[270.367,125.905975],[228.74477,124.23451],[186.46698,122.41155],[143.28162,120.50214],[99.135445,118.67451]],"6.png":[[205.1374,338.7553],[183.50075,340.3901],[161.20695,342.18256],[137.93665,344.07538],[113.66636,346.13974],[204.64433,315.9385],[183.20177,317.2081],[160.94232,318.50116],[137.79437,319.92603],[113.66841,321.53873],[204.21393,293.48245],[182.77318,294.34222],[160.64436,295.1879],[137.6513,296.23132],[113.58201,297.30646],[203.78062,271.24112],[182.49759,271.59387],[160.42705,272.15204],[137.47346,272.61227],[113.47126,273.26758],[203.47084,249.04837],[182.21783,249.14967],[160.13246,249.22328],[137.22206,249.27446],[113.23081,249.31792],[203.15816,226.85248],[181.74883,226.58026],[159.71736,226.30402],[136.78018,225.7971],[112.76668,225.38472],[202.70686,204.63266],[181.44064,204.03471],[159.40121,203.30002],[136.3704,202.38806],[112.39394,201.39375],[202.43837,182.36055],[181.11613,181.31291],[158.86263,180.12228],[135.83168,178.72948],[111.77687,177.34703]],"3.png":[[299.15402,347.75366],[252.70624,351.49863],[203.43675,355.3683],[150.6566,360.01315],[93.47113,365.1607],[280.4798,309.10272],[237.32909,311.15698],[192.24583,313.51416],[144.1358,316.3689],[91.82277,319.6175],[264.55145,276.3868],[224.55254,277.5483],[182.84058,278.8685],[138.55806,280.40747],[90.540955,282.01904],[251.17128,248.51814],[213.902,249.18242],[175.06148,249.51427],[133.68584,249.96053],[89.2959,250.35948],[239.60036,224.46333],[204.64624,224.46748],[168.23434,224.25208],[129.4866,223.69078],[88.00073,223.31033],[229.5496,203.2498],[196.54051,202.67274],[162.1461,201.94096],[125.63078,200.79773],[86.851036,199.56558],[220.67259,184.37373],[189.39381,183.42145],[156.68655,182.18793],[122.2751,180.5192],[85.79754,178.82803],[212.7082,167.38385],[182.95573,166.10435],[151.83192,164.40715],[119.28999,162.52534],[85.03209,160.61536]],"1.png":[[258.19308,410.6701],[224.21643,413.52533],[189.52975,416.45392],[154.45471,419.35205],[118.751785,422.2567],[262.28873,388.9599],[226.40807,391.62018],[190.02585,394.44928],[153.01097,397.44476],[115.46515,400.40863],[266.58188,364.6367],[228.761,367.13306],[190.51605,369.72098],[151.5804,372.7239],[111.71821,375.9016],[271.37625,337.50485],[231.38153,339.6239],[191.16032,342.189],[150.0261,345.12607],[107.889114,348.35907],[276.54138,307.0877],[234.25636,309.15213],[191.64252,311.49857],[148.41096,314.30542],[103.52165,317.362],[282.56787,272.9135],[237.52971,274.9331],[192.45366,277.32013],[146.43877,279.8038],[98.66458,282.57462],[289.6469,234.26715],[241.47043,236.38443],[193.31143,238.5409],[144.13129,240.70642],[92.735115,242.95776],[298.02255,189.57112],[246.36998,191.65927],[194.32751,193.75522],[141.13118,195.47801],[85.6814,196.92738]],"5.png":[[222.92903,428.0741],[180.524,418.14813],[142.56943,409.30167],[108.49659,401.2517],[77.66134,393.95355],[221.38434,379.97888],[178.65211,372.46667],[140.63782,365.94193],[106.357666,360.36377],[75.28938,355.15347],[219.50801,330.42737],[176.94487,325.63705],[138.8906,321.78653],[104.45778,318.48126],[73.07517,315.48795],[217.6895,280.70447],[175.21082,279.0187],[137.13873,277.55502],[102.49359,276.44534],[70.82157,275.35394],[216.23053,230.64198],[173.34407,231.9026],[134.998,233.01358],[100.174995,233.78107],[68.402176,234.61082],[214.74593,179.09714],[171.38005,183.51566],[132.5897,187.22311],[97.48024,190.37315],[65.67558,193.29755],[213.43802,125.577805],[169.33946,133.3386],[130.02704,139.97592],[94.750946,145.79755],[63.26309,151.27016],[211.727,71.84391],[167.37796,82.71496],[127.76215,92.38475],[92.52531,100.978294],[60.797474,108.67862]],"4.png":[[225.3423,371.51007],[202.30902,379.46854],[175.96132,388.6032],[145.57164,399.19708],[110.24223,411.23743],[224.53914,336.4252],[201.56935,341.8711],[175.41623,348.34198],[145.09813,355.95657],[109.497894,364.97342],[223.70544,301.69366],[200.76765,304.83615],[174.77408,308.57504],[144.6994,313.0564],[109.254906,318.3582],[223.09134,267.58868],[200.26772,268.53745],[174.311,269.5642],[144.31773,270.81934],[108.79843,272.42438],[222.48608,233.659],[199.59637,232.44116],[173.62344,230.89288],[143.5836,229.1122],[108.20664,226.78975],[221.90434,199.60367],[199.00636,196.16911],[172.8986,191.96538],[142.69672,187.03201],[107.185326,181.05688],[221.48347,165.16344],[198.42995,159.32457],[172.23988,152.45059],[141.74673,144.36873],[106.2948,135.00398],[221.05043,130.43988],[197.78748,122.23632],[171.47011,112.77497],[141.15128,102.03716],[105.707886,89.57386]]}
Loading

0 comments on commit 5c0db73

Please sign in to comment.