Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

issue:207 Support redis sentinel mode. #241

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 11 additions & 5 deletions src/main/scala/com/redislabs/provider/redis/ConnectionPool.scala
Original file line number Diff line number Diff line change
@@ -1,16 +1,17 @@
package com.redislabs.provider.redis

import redis.clients.jedis.{JedisPoolConfig, Jedis, JedisPool}
import redis.clients.jedis.{Jedis, JedisPool, JedisPoolConfig, JedisSentinelPool}
import redis.clients.jedis.exceptions.JedisConnectionException

import java.util.concurrent.ConcurrentHashMap

import redis.clients.jedis.util.Pool

import scala.collection.JavaConversions._


object ConnectionPool {
@transient private lazy val pools: ConcurrentHashMap[RedisEndpoint, JedisPool] =
new ConcurrentHashMap[RedisEndpoint, JedisPool]()
@transient private lazy val pools: ConcurrentHashMap[RedisEndpoint, Pool[Jedis]] =
new ConcurrentHashMap[RedisEndpoint, Pool[Jedis]]()

def connect(re: RedisEndpoint): Jedis = {
val pool = pools.getOrElseUpdate(re,
Expand All @@ -25,7 +26,12 @@ object ConnectionPool {
poolConfig.setTimeBetweenEvictionRunsMillis(30000)
poolConfig.setNumTestsPerEvictionRun(-1)

new JedisPool(poolConfig, re.host, re.port, re.timeout, re.auth, re.dbNum, re.ssl)
if (null == re.master || re.master.trim.isEmpty) {
new JedisPool(poolConfig, re.host, re.port, re.timeout, re.auth, re.dbNum, re.ssl)
} else {
val sentinels = re.host.split(",").map(x => x + ":" + re.port).toSet
new JedisSentinelPool(re.master.trim, sentinels, poolConfig, re.auth)
}
}
)
var sleepTime: Int = 4
Expand Down
12 changes: 7 additions & 5 deletions src/main/scala/com/redislabs/provider/redis/RedisConfig.scala
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,8 @@ case class RedisEndpoint(host: String = Protocol.DEFAULT_HOST,
auth: String = null,
dbNum: Int = Protocol.DEFAULT_DATABASE,
timeout: Int = Protocol.DEFAULT_TIMEOUT,
ssl: Boolean = false)
ssl: Boolean = false,
master: String = null)
extends Serializable {

/**
Expand All @@ -39,7 +40,8 @@ case class RedisEndpoint(host: String = Protocol.DEFAULT_HOST,
conf.get("spark.redis.auth", null),
conf.getInt("spark.redis.db", Protocol.DEFAULT_DATABASE),
conf.getInt("spark.redis.timeout", Protocol.DEFAULT_TIMEOUT),
conf.getBoolean("spark.redis.ssl", false)
conf.getBoolean("spark.redis.ssl", false),
conf.get("spark.redis.sentinel.master", null)
)
}

Expand Down Expand Up @@ -254,7 +256,7 @@ class RedisConfig(val initialHost: RedisEndpoint) extends Serializable {

//simply re-enter this function witht he master host/port
getNonClusterNodes(initialHost = new RedisEndpoint(host, port,
initialHost.auth, initialHost.dbNum, ssl = initialHost.ssl))
initialHost.auth, initialHost.dbNum, ssl = initialHost.ssl, master = initialHost.master))

} else {
//this is a master - take its slaves
Expand All @@ -270,7 +272,7 @@ class RedisConfig(val initialHost: RedisEndpoint) extends Serializable {
val range = nodes.length
(0 until range).map(i =>
RedisNode(RedisEndpoint(nodes(i)._1, nodes(i)._2, initialHost.auth, initialHost.dbNum,
initialHost.timeout, initialHost.ssl),
initialHost.timeout, initialHost.ssl, initialHost.master),
0, 16383, i, range)).toArray
}
}
Expand Down Expand Up @@ -300,7 +302,7 @@ class RedisConfig(val initialHost: RedisEndpoint) extends Serializable {
val host = SafeEncoder.encode(node.get(0).asInstanceOf[Array[scala.Byte]])
val port = node.get(1).toString.toInt
RedisNode(RedisEndpoint(host, port, initialHost.auth, initialHost.dbNum,
initialHost.timeout, initialHost.ssl),
initialHost.timeout, initialHost.ssl, initialHost.master),
sPos,
ePos,
i,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,8 @@ class RedisSourceRelation(override val sqlContext: SQLContext,
val dbNum = parameters.get("dbNum").map(_.toInt).getOrElse(Protocol.DEFAULT_DATABASE)
val timeout = parameters.get("timeout").map(_.toInt).getOrElse(Protocol.DEFAULT_TIMEOUT)
val ssl = parameters.get("ssl").map(_.toBoolean).getOrElse(false)
RedisEndpoint(host, port, auth, dbNum, timeout, ssl)
val master = parameters.getOrElse("sentinel.master", null)
RedisEndpoint(host, port, auth, dbNum, timeout, ssl, master)
}
)
}
Expand Down