Skip to content

update rebar to 3.18.0; use https:// when fetching deps from github #227

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 7 commits into
base: develop-3.0
Choose a base branch
from
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 1 addition & 2 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,11 +1,10 @@
_build/
rebar.lock
ebin/
deps/
tests/
/basho_bench
/rel/basho_bench
package
.rebar
*~
#*#
.DS_Store
60 changes: 7 additions & 53 deletions Makefile
Original file line number Diff line number Diff line change
@@ -6,37 +6,19 @@ PKG_ID = basho-bench-$(PKG_VERSION)
PKG_BUILD = 1
BASE_DIR = $(shell pwd)
ERLANG_BIN = $(shell dirname $(shell which erl))
REBAR ?= $(BASE_DIR)/rebar
REBAR ?= $(BASE_DIR)/rebar3
OVERLAY_VARS ?=


all: deps compile
$(REBAR) skip_deps=true escriptize
all: compile
$(REBAR) escriptize

.PHONY: deps compile rel lock locked-all locked-deps
.PHONY: compile rel

rel: deps compile
cd rel && $(REBAR) generate skip_deps=true $(OVERLAY_VARS)
rel: compile
cd rel && $(REBAR) generate $(OVERLAY_VARS)

deps:
$(REBAR) get-deps

##
## Lock Targets
##
## see https://github.com/seth/rebar_lock_deps_plugin
lock: deps compile
$(REBAR) lock-deps

locked-all: locked-deps compile

locked-deps:
@echo "Using rebar.config.lock file to fetch dependencies"
$(REBAR) -C rebar.config.lock get-deps

compile: deps
# Temp hack to work around https://github.com/basho/riak-erlang-client/issues/151
(cd deps/riak_pb ; $(REBAR) clean compile deps_dir=..)
compile:
@($(REBAR) compile)

clean:
@@ -74,31 +56,3 @@ JOBS := $(addprefix job,${TARGETS})

all_results: ${JOBS} ; echo "$@ successfully generated."
${JOBS}: job%: ; Rscript --vanilla priv/summary.r -i tests/$*

##
## Packaging targets
##
.PHONY: package
export PKG_VERSION PKG_ID PKG_BUILD BASE_DIR ERLANG_BIN REBAR OVERLAY_VARS RELEASE

package.src: deps
mkdir -p package
rm -rf package/$(PKG_ID)
git archive --format=tar --prefix=$(PKG_ID)/ $(PKG_REVISION)| (cd package && tar -xf -)
${MAKE} -C package/$(PKG_ID) locked-deps
for dep in package/$(PKG_ID)/deps/*; do \
echo "Processing dep: $${dep}"; \
mkdir -p $${dep}/priv; \
git --git-dir=$${dep}/.git describe --always --tags >$${dep}/priv/vsn.git; \
done
find package/$(PKG_ID) -depth -name ".git" -exec rm -rf {} \;
tar -C package -czf package/$(PKG_ID).tar.gz $(PKG_ID)

dist: package.src
cp package/$(PKG_ID).tar.gz .

package: package.src
${MAKE} -C package -f $(PKG_ID)/deps/node_package/Makefile

pkgclean: distclean
rm -rf package
10 changes: 5 additions & 5 deletions include/basho_bench.hrl
Original file line number Diff line number Diff line change
@@ -2,12 +2,12 @@
-define(FAIL_MSG(Str, Args), ?ERROR(Str, Args), basho_bench_app:stop_or_kill()).
-define(STD_ERR(Str, Args), io:format(standard_error, Str, Args)).

-define(CONSOLE(Str, Args), lager:info(Str, Args)).
-define(CONSOLE(Str, Args), logger:info(Str, Args)).

-define(DEBUG(Str, Args), lager:debug(Str, Args)).
-define(INFO(Str, Args), lager:info(Str, Args)).
-define(WARN(Str, Args), lager:warning(Str, Args)).
-define(ERROR(Str, Args), lager:error(Str, Args)).
-define(DEBUG(Str, Args), logger:debug(Str, Args)).
-define(INFO(Str, Args), logger:info(Str, Args)).
-define(WARN(Str, Args), logger:warning(Str, Args)).
-define(ERROR(Str, Args), logger:error(Str, Args)).

-define(FMT(Str, Args), lists:flatten(io_lib:format(Str, Args))).

24 changes: 0 additions & 24 deletions pkg.vars.config

This file was deleted.

26 changes: 8 additions & 18 deletions rebar.config
Original file line number Diff line number Diff line change
@@ -1,31 +1,21 @@
{require_otp_vsn, "20"}.
%% -*- mode: erlang -*-
{require_otp_vsn, "22"}.

{deps,
[
{folsom, "0.8.7"},
{lager, "2.*", {git, "git://github.com/erlang-lager/lager", {tag, "3.8.0"}}},
{ibrowse, ".*",
{git, "git://github.com/basho/ibrowse.git", {branch, "develop-3.0"}}},
{riakc, ".*",
{git, "git://github.com/basho/riak-erlang-client", {branch, "develop-3.0"}}},
{mochiweb, "2.9.*",
{git, "git://github.com/basho/mochiweb", {branch, "develop-3.0"}}},
{getopt, ".*",
{git, "git://github.com/jcomellas/getopt", {tag, "v0.8.2"}}},
{bitcask, ".*",
{git, "git://github.com/basho/bitcask", {branch, "develop-3.0"}}},
{eleveldb, ".*",
{git, "git://github.com/basho/eleveldb", {branch, "develop-3.0"}}}
{ibrowse, {git, "https://github.com/basho/ibrowse.git", {branch, "develop-3.0"}}},
{riakc, {git, "https://github.com/basho/riak-erlang-client", {branch, "develop"}}},
{mochiweb, {git, "https://github.com/basho/mochiweb", {branch, "develop-3.0"}}},
{getopt, {git, "https://github.com/jcomellas/getopt", {tag, "v0.8.2"}}},
{bitcask, {git, "https://github.com/basho/bitcask", {branch, "develop"}}},
{eleveldb, {git, "https://github.com/basho/eleveldb", {branch, "develop"}}}
]}.

{erl_opts, [{src_dirs, [src]},
{parse_transform, lager_transform}]}.

{escript_incl_apps, [
folsom,
getopt,
ibrowse,
lager,
mochiweb,
riakc,
bitcask,
50 changes: 0 additions & 50 deletions rebar.lock

This file was deleted.

Binary file modified rebar3
Binary file not shown.
63 changes: 0 additions & 63 deletions rel/reltool.config

This file was deleted.

11 changes: 5 additions & 6 deletions src/basho_bench.app.src
Original file line number Diff line number Diff line change
@@ -5,14 +5,13 @@
{registered, [ basho_bench_sup ]},
{applications, [kernel,
stdlib,
sasl,
lager]},
sasl]},
{mod, {basho_bench_app, []}},
{env, [
%% Run mode: How should basho_bench started as a separate node, or part of an
%% other node. The default is standalone, other option is included.
{app_run_mode, standalone},
%% Run mode: How should basho_bench started as a separate node, or part of an
%% other node. The default is standalone, other option is included.
{app_run_mode, standalone},

%%
%% Mode of load generation:
%% max - Generate as many requests as possible per worker
38 changes: 10 additions & 28 deletions src/basho_bench.erl
Original file line number Diff line number Diff line change
@@ -23,6 +23,7 @@

-export([main/1, md5/1]).
-include("basho_bench.hrl").
-include_lib("kernel/include/logger.hrl").

%% ====================================================================
%% API
@@ -56,29 +57,10 @@ main(Args) ->
{ok, _Pid} = basho_bench_config:start_link(),
basho_bench_config:set(test_id, BenchName),

application:load(lager),
ConsoleLagerLevel = basho_bench_config:get(log_level, debug),
ErrorLog = filename:join([TestDir, "error.log"]),
ConsoleLog = filename:join([TestDir, "console.log"]),
CrashLog = filename:join([TestDir, "crash.log"]),
application:set_env(lager,
handlers,
[{lager_console_backend, ConsoleLagerLevel},
{lager_file_backend, [{file, ErrorLog}, {level, error}, {size, 10485760}, {date, "$D0"}, {count, 5}]},
{lager_file_backend, [{file, ConsoleLog}, {level, debug}, {size, 10485760}, {date, "$D0"}, {count, 5}]}
]),
application:set_env(lager, crash_log, CrashLog),
lager:start(),

%% Make sure this happens after starting lager or failures wont
%% show.
basho_bench_config:load(Configs),

%% Log level can be overriden by the config files
CustomLagerLevel = basho_bench_config:get(log_level),
lager:set_loglevel(lager_console_backend, CustomLagerLevel),
lager:set_loglevel(lager_file_backend, ConsoleLog, CustomLagerLevel),

%% Init code path
add_code_paths(basho_bench_config:get(code_paths, [])),

@@ -94,7 +76,7 @@ main(Args) ->
%% Copy the config into the test dir for posterity
[ begin {ok, _} = file:copy(Config, filename:join(TestDir, filename:basename(Config))) end
|| Config <- Configs ],
case basho_bench_config:get(distribute_work, false) of
case basho_bench_config:get(distribute_work, false) of
true -> setup_distributed_work();
false -> ok
end,
@@ -288,8 +270,8 @@ get_addr_args() ->
StrAddrs = [inet:ntoa(Addr) || Addr <- Addrs],
string:join(StrAddrs, " ").
setup_distributed_work() ->
case node() of
'nonode@nohost' ->
case node() of
'nonode@nohost' ->
?STD_ERR("Basho bench not started in distributed mode, and distribute_work = true~n", []),
halt(1);
_ -> ok
@@ -307,13 +289,13 @@ setup_distributed_work() ->
[pool:attach(SlaveName) || SlaveName <- SlaveNames],
CodePaths = code:get_path(),
rpc:multicall(SlaveNames, code, set_path, [CodePaths]),
Apps = [lager, basho_bench, getopt, bear, folsom, ibrowse, riakc, riak_pb, mochiweb, protobuffs, goldrush],
Apps = [basho_bench, getopt, bear, folsom, ibrowse, riakc, riak_pb, mochiweb, protobuffs, goldrush],
[distribute_app(App) || App <- Apps].


deploy_module(Module) ->
case basho_bench_config:get(distribute_work, false) of
true ->
case basho_bench_config:get(distribute_work, false) of
true ->
Nodes = nodes(),
{Module, Binary, Filename} = code:get_object_code(Module),
rpc:multicall(Nodes, code, load_binary, [Module, Filename, Binary]);
@@ -330,19 +312,19 @@ distribute_app(App) ->
EbinsDir = lists:filter(fun(CodePathDir) -> string:substr(CodePathDir, 1, LibDirLen) == LibDir end, code:get_path()),
StripEndFun = fun(Path) ->
PathLen = string:len(Path),
case string:substr(Path, PathLen - string:len(CodeExtension) + 1, string:len(Path)) of
case string:substr(Path, PathLen - string:len(CodeExtension) + 1, string:len(Path)) of
CodeExtension ->
{true, string:substr(Path, 1, PathLen - string:len(CodeExtension))};
_ -> false
end
end,
end,
EbinDirDistributeFun = fun(EbinDir) ->
{ok, Beams} = erl_prim_loader:list_dir(EbinDir),
Modules = lists:filtermap(StripEndFun, Beams),
ModulesLoaded = [code:load_abs(filename:join(EbinDir, ModFileName)) || ModFileName <- Modules],
lists:foreach(fun({module, Module}) -> deploy_module(Module) end, ModulesLoaded)
end,
lists:foreach(EbinDirDistributeFun, EbinsDir),
lists:foreach(EbinDirDistributeFun, EbinsDir),
ok.
%% just a utility, should be in basho_bench_utils.erl
%% but 's' is for multiple utilities, and so far this
8 changes: 5 additions & 3 deletions src/basho_bench_driver_cluster.erl
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
%% -------------------------------------------------------------------
%%
%% basho_bench: benchmark service on any within Erlang cluster
%% basho_bench: benchmark service on any within Erlang cluster
%% using distribution protocol
%%
%% Copyright (c) 2015 Dmitry Kolesnikov
@@ -27,6 +27,8 @@
run/4
]).

-include_lib("kernel/include/logger.hrl").

-record(state, {actor}).

%% ====================================================================
@@ -39,11 +41,11 @@ new(Id) ->
{Name, Node} = Actor = lists:nth(Nth, Actors),
case net_adm:ping(Node) of
pang ->
lager:error("~s is not available", [Node]),
logger:error("~s is not available", [Node]),
{ok, #state{actor = undefined}};

pong ->
lager:info("worker ~b is bound to ~s on ~s", [Id, Name, Node]),
logger:info("worker ~b is bound to ~s on ~s", [Id, Name, Node]),
{ok, #state{actor = Actor}}
end.

28 changes: 15 additions & 13 deletions src/basho_bench_driver_cs.erl
Original file line number Diff line number Diff line change
@@ -26,6 +26,8 @@

-module(basho_bench_driver_cs).

-include_lib("kernel/include/logger.hrl").

-define(BLOCK, (1024*1024)).
-define(VERYLONG_TIMEOUT, (300*1000)).
-define(REPORTFUN_APPKEY, report_fun).
@@ -76,15 +78,15 @@ new(ID) ->
N = mb_sec -> {N, fun(X) -> X / (1000 * 1000) end};
N = mib_sec -> {N, fun(X) -> X / (1024 * 1024) end};
_ ->
lager:log(
logger:log(
error, self(),
"Unrecognized value for cs_measurement_units.\n", []),
exit(unrecognized_value)
end,
if ID == 1 ->
application:start(ibrowse),
application:start(crypto),
lager:log(info, self(), "Reporting factor = ~p\n", [RF_name]);
logger:log(info, self(), "Reporting factor = ~p\n", [RF_name]);
true ->
ok
end,
@@ -99,7 +101,7 @@ new(ID) ->
lists:keymember(delete, 1, OpsList) andalso
length(OpsList) > 1 of
true ->
lager:log(
logger:log(
warning, self(),
"Mixing delete and non-delete operations together with "
"~p measurements unit can yield nonsense results!\n\n",
@@ -123,9 +125,9 @@ new(ID) ->
{ProxyH, ProxyP} = lists:nth((ID rem length(ProxyTargets)+1), ProxyTargets),
ID_max = 30,
if ID == ID_max ->
lager:log(info, self(), "Suppressing additional proxy info", []);
logger:log(info, self(), "Suppressing additional proxy info", []);
ID < ID_max ->
lager:log(info, self(), "ID ~p Proxy host ~p TCP port ~p\n",
logger:log(info, self(), "ID ~p Proxy host ~p TCP port ~p\n",
[ID, ProxyH, ProxyP]);
true ->
ok
@@ -253,7 +255,7 @@ perhaps_sleep() ->

bigfile_valgen(Id, Props) ->
if Id == 1 ->
lager:log(info, self(), "~p value gen props: ~p\n", [?MODULE, Props]);
logger:log(info, self(), "~p value gen props: ~p\n", [?MODULE, Props]);
true ->
ok
end,
@@ -312,7 +314,7 @@ insert(KeyGen, ValueGen, {Host, Port}, Bucket, State) ->
%% tell us its content length.
{ValueGen, ValFunc(get_content_length)};
_ ->
lager:log(
logger:log(
error, self(),
"This driver cannot use the standard basho_bench "
"generator functions, please see refer to "
@@ -615,7 +617,7 @@ setup_user_and_bucket(State) ->
DisplayName = basho_bench_config:get(cs_display_name, "test-user"),
ok = maybe_create_user(DisplayName, State),
{ok, {_DisplayName, KeyId, KeySecret}} = fetch_user_info(DisplayName, State),
lager:info("Target User: ~p", [{DisplayName, KeyId, KeySecret}]),
logger:info("Target User: ~p", [{DisplayName, KeyId, KeySecret}]),
ok = basho_bench_config:set(cs_access_key, KeyId),
ok = basho_bench_config:set(cs_secret_key, KeySecret);
_ ->
@@ -631,10 +633,10 @@ maybe_create_user(DisplayName, #state{hosts=Hosts} = State) ->
Headers = [{'Content-Type', 'application/json'}],
case send_request({Host, Port}, Url, Headers, post, Json, proxy_opts(State)) of
{ok, "201", _Header, Body} ->
lager:debug("User created: ~p~n", [Body]),
logger:debug("User created: ~p~n", [Body]),
ok;
{ok, "409", _Header, Body} ->
lager:debug("User already exists: ~p~n", [Body]),
logger:debug("User already exists: ~p~n", [Body]),
ok;
{ok, Code, Header, Body} ->
{error, {user_creation, Code, Header, Body}};
@@ -688,12 +690,12 @@ maybe_create_bucket(Bucket, #state{hosts=Hosts} = State) ->
Url = url(Host, Port, Bucket, undefined),
case send_request({Host, Port}, Url, [], put, [], proxy_opts(State)) of
{ok, "200", _Headers, _Body} ->
lager:debug("Bucket created (maybe): ~p~n", [Bucket]),
logger:debug("Bucket created (maybe): ~p~n", [Bucket]),
ok;
{ok, Code, Header, Body} ->
lager:error("Create bucket: ~p~n", [{Code, Header, Body}]),
logger:error("Create bucket: ~p~n", [{Code, Header, Body}]),
{error, {bucket_creation, Code, Header, Body}};
{error, Reason} ->
lager:error("Create bucket: ~p~n", [Reason]),
logger:error("Create bucket: ~p~n", [Reason]),
{error, {bucket_creation, Reason}}
end.
239 changes: 120 additions & 119 deletions src/basho_bench_driver_nhs.erl

Large diffs are not rendered by default.

35 changes: 18 additions & 17 deletions src/basho_bench_driver_riakc_pb.erl
Original file line number Diff line number Diff line change
@@ -29,6 +29,7 @@
-export([run_listkeys/1]).

-include("basho_bench.hrl").
-include_lib("kernel/include/logger.hrl").

-record(state, { pid,
bucket,
@@ -199,7 +200,7 @@ run({team, write}, KeyGen, _ValueGen, State) ->
{ok, _} ->
{ok, State};
{error, Reason} ->
lager:info("Team write failed, error: ~p", [Reason]),
logger:info("Team write failed, error: ~p", [Reason]),
{error, Reason, State}
end;

@@ -215,10 +216,10 @@ run({team, read}, KeyGen, ValueGen, State) ->
{ok, _} ->
{ok, State};
{error, {notfound, _}} ->
lager:info("Team does not exist yet."),
logger:info("Team does not exist yet."),
run({team, write}, KeyGen, ValueGen, State);
{error, Reason} ->
lager:info("Team read failed, error: ~p", [Reason]),
logger:info("Team read failed, error: ~p", [Reason]),
{error, Reason, State}
end;

@@ -254,17 +255,17 @@ run({team, player, removal}, KeyGen, ValueGen, State) ->
{ok, _} ->
{ok, State};
{error, Reason} ->
lager:info("Team player removal failed, error: ~p", [Reason]),
logger:info("Team player removal failed, error: ~p", [Reason]),
{error, Reason, State}
end;
false ->
{ok, State}
end;
{error, {notfound, _}} ->
lager:info("Team does not exist yet."),
logger:info("Team does not exist yet."),
run({team, write}, KeyGen, ValueGen, State);
{error, Reason} ->
lager:info("Team read failed, error: ~p", [Reason]),
logger:info("Team read failed, error: ~p", [Reason]),
{error, Reason, State}
end;

@@ -287,7 +288,7 @@ run({team, player, addition}, KeyGen, ValueGen, State) ->
{ok, _} ->
{ok, State};
{error, Reason} ->
lager:info("Team player addition failed, error: ~p", [Reason]),
logger:info("Team player addition failed, error: ~p", [Reason]),
{error, Reason, State}
end;

@@ -311,7 +312,7 @@ run({game, completed}, KeyGen, ValueGen, State) ->
{ok, _} ->
{ok, State};
{error, Reason} ->
lager:info("Score change failed, error: ~p", [Reason]),
logger:info("Score change failed, error: ~p", [Reason]),
{error, Reason, State}
end;

@@ -505,7 +506,7 @@ run(listkeys, _KeyGen, _ValueGen, State) ->
end,
case {State#state.nominated_id, IsAlive} of
{true, true} ->
lager:info("Skipping listkeys for overlap"),
logger:info("Skipping listkeys for overlap"),
{ok, State};
{true, false} ->
Pid = spawn(?MODULE, run_listkeys, [State]),
@@ -563,7 +564,7 @@ run(mr_keylist_js, KeyGen, _ValueGen, State) ->

run({counter, value}, KeyGen, _ValueGen, State) ->
Key = KeyGen(),
lager:info("Counter value called for key: ~p", [Key]),
logger:info("Counter value called for key: ~p", [Key]),
Options = [{r,2}, {notfound_ok, true}, {timeout, 5000}],
Result = riakc_pb_socket:fetch_type(State#state.pid,
State#state.bucket,
@@ -572,19 +573,19 @@ run({counter, value}, KeyGen, _ValueGen, State) ->
case Result of
{ok, C0} ->
C = riakc_counter:value(C0),
lager:info("Counter value is: ~p", [C]),
logger:info("Counter value is: ~p", [C]),
{ok, State};
{error, {notfound, _}} ->
{ok, State};
{error, Reason} ->
lager:info("Team read failed, error: ~p", [Reason]),
logger:info("Team read failed, error: ~p", [Reason]),
{error, Reason, State}
end;

run({counter, increment}, KeyGen, ValueGen, State) ->
Amt = ValueGen(),
Key = KeyGen(),
lager:info("Counter value called for key: ~p", [Key]),
logger:info("Counter value called for key: ~p", [Key]),
Result = riakc_pb_socket:modify_type(State#state.pid,
fun(C) ->
riakc_counter:increment(Amt, C)
@@ -596,7 +597,7 @@ run({counter, increment}, KeyGen, ValueGen, State) ->
{ok, _} ->
{ok, State};
{error, Reason} ->
lager:info("Counter increment failed, error: ~p", [Reason]),
logger:info("Counter increment failed, error: ~p", [Reason]),
{error, Reason, State}
end;

@@ -799,7 +800,7 @@ record_2i_results(Results, State) ->

run_listkeys(State) ->
SW = os:timestamp(),
lager:info("Commencing listkeys request"),
logger:info("Commencing listkeys request"),

Targets = State#state.singleton_targets,
{TargetIp, TargetPort} = lists:nth(rand:uniform(length(Targets)+1),
@@ -813,12 +814,12 @@ run_listkeys(State) ->
State#state.bucket,
State#state.timeout_listkeys) of
{ok, Keys} ->
lager:info("listkeys request returned ~w keys" ++
logger:info("listkeys request returned ~w keys" ++
" in ~w seconds",
[length(Keys),
timer:now_diff(os:timestamp(), SW)/1000000]),
ok;
{error, Reason} ->
lager:info("listkeys failed due to reason ~w", [Reason]),
logger:info("listkeys failed due to reason ~w", [Reason]),
ok
end.
6 changes: 3 additions & 3 deletions src/basho_bench_worker.erl
Original file line number Diff line number Diff line change
@@ -53,10 +53,10 @@
%% ====================================================================

start_link(SupChild, Id) ->
case basho_bench_config:get(distribute_work, false) of
true ->
case basho_bench_config:get(distribute_work, false) of
true ->
start_link_distributed(SupChild, Id);
false ->
false ->
start_link_local(SupChild, Id)
end.