Codis 集群搭建

整体安装步骤
1、安装依赖组件 Go、Godep、Python、Zookeeper;
2、安装 Codis 本身组件;
3、配置 Codis 集群,将组件关联到集群中。

1、安装依赖组件

安装 Go、Godep、Python、Zookeeper

Go 语言安装

cd /opt
wget https://dl.google.com/go/go1.11.linux-amd64.tar.gz
mkdir gopath
tar -zxvf go1.11.linux-amd64.tar.gz

#环境变量
vim ~/.bash_profile
  export GOROOT=/opt/go
  export GOPATH=/opt/gopath
  export PATH=${GOROOT}/bin:$PATH 
  export PATH=${GOPATH}/bin:$PATH

#检查版本
source ~/.bash_profile
go version

Godep 安装

mkdir -p  /opt/gopath/src/github.com/tools/
cd /opt/gopath/src/github.com/tools/

yum install git -y
git clone https://github.com/tools/godep.git
cd godep/
#该命令会将 godep 生成到${GOPATH}/bin 下
go install ./

Python 安装

#安装依赖包
yum -y groupinstall "Development tools" 
yum -y install zlib-devel bzip2-devel openssl-devel ncurses-devel sqlite-devel readline-devel tk-devel gdbm-devel db4-devel libpcap-devel xz-devel

#安装python3.7
mkdir -p /opt/python/source/
cd /opt/python/source/
wget https://www.python.org/ftp/python/3.7.6/Python-3.7.6.tar.xz
tar -xvf  Python-3.7.6.tar.xz
cd Python-3.7.6/
./configure --prefix=/opt/python
make && make install
ln -s /opt/python/bin/python3 /usr/bin/python3
ln -s /opt/python/bin/pip3 /usr/bin/pip3

#验证
python3 --version

Zookeeper 安装

mkdir  /opt/zookeeper
cd /opt/zookeeper

wget http://mirror.bit.edu.cn/apache/zookeeper/zookeeper-3.4.14/zookeeper-3.4.14.tar.gz

tar -zxvf zookeeper-3.4.14.tar.gz 
cd zookeeper-3.4.14/conf
cp zoo_sample.cfg  zoo.cfg

#启动
cd /opt/zookeeper/zookeeper-3.4.14/bin
./zkServer.sh start

#检查状态
./zkServer.sh status

/opt/zookeeper/zookeeper-3.4.14/bin/zkCli.sh
ls /jodis/codis-demo

2、Codis 自身组件安装

下载 codis

cd /opt/gopath/src/github.com
mkdir CodisLabs
cd CodisLabs
git clone https://github.com/CodisLabs/codis.git -b release3.2

vim ~/.bash_profile 
  export CODIS_HOME=${GOPATH}/src/github.com/CodisLabs/codis
  export PATH=${CODIS_HOME}/bin:$PATH

编译

source ~/.bash_profile 
cd ${CODIS_HOME} 
find . -name '*.sh' | xargs chmod +x
find . -name '*.py' | xargs chmod +x
#文件可能为格式为Windows格式,需改成UNIX格式
yum install -y dos2unix
find . -name '*.sh' | xargs dos2unix
find . -name 'version' | xargs dos2unix
find . -name '*.py' | xargs dos2unix

#编译
make MALLOC=libc
make gotest

dashboard 配置

cd ${CODIS_HOME}/config
cp dashboard.toml dashboard-18080.toml
vim dashboard-18080.toml
  coordinator_name = "zookeeper"
  coordinator_addr = "127.0.0.1:2181"
  product_name = "codis-demo"
  product_auth = "demo"

dashboard 启动脚本

mkdir -p /opt/codis
cd /opt/codis
vim dashboard-start.sh
  cd ${CODIS_HOME}/bin
  PORT=18080
  NCPU=4
  CONFIG=${CODIS_HOME}/config/dashboard-${PORT}.toml
  LOG=${CODIS_HOME}/logs/dashboard-${PORT}.log
  PID=${CODIS_HOME}/logs/dashboard-${PORT}.pid
  LEVEL="WARN"
  codis-dashboard --ncpu=${NCPU} --config=${CONFIG} --log=${LOG} --pidfile=${PID} --log-level=${LEVEL} &

dashboard 关闭脚本

cd /opt/codis
vim dashboard-stop.sh
  PORT=18080
  PID=${CODIS_HOME}/logs/dashboard-${PORT}.pid
  cat ${PID} | xargs kill

proxy 配置

cd ${CODIS_HOME}/config
cp proxy.toml proxy-11080.toml
vim proxy-11080.toml
  product_name = "codis-demo"
  product_auth = "demo"
  # session_auth 客户端jodis连接密码【jodis代码连接使用密码】
  session_auth = "codis-demo"
  admin_addr = "0.0.0.0:11080"
  proxy_addr = "0.0.0.0:19000"
  jodis_name = "zookeeper"
  jodis_addr = "127.0.0.1:2181"
  # 与redis中databases保持一致
  backend_number_databases = 1

proxy 启动脚本

cd /opt/codis
vim /opt/codis/proxy-start.sh
  cd ${CODIS_HOME}/bin
  PORT=11080
  NCPU=4
  CONFIG=${CODIS_HOME}/config/proxy-${PORT}.toml
  LOG=${CODIS_HOME}/logs/proxy-${PORT}.log
  PID=${CODIS_HOME}/logs/proxy-${PORT}.pid
  LEVEL="WARN"
  codis-proxy --ncpu=${NCPU} --config=${CONFIG} --log=${LOG} --pidfile=${PID} --log-level=${LEVEL} &

proxy 关闭脚本

vim /opt/codis/proxy-stop.sh
  PORT=11080
  PID=${CODIS_HOME}/logs/proxy-${PORT}.pid
  cat ${PID} | xargs kill

server [主] 配置

#每个节点上有一个主和一个备
#主
mkdir -p ${CODIS_HOME}/data 
cd ${CODIS_HOME}/config
vim ${CODIS_HOME}/config/server-master-6379.conf
  bind 0.0.0.0
  port 6379
  dir /opt/gopath/src/github.com/CodisLabs/codis/data
  pidfile ../logs/redis-6379.pid
  logfile ../logs/redis-6379.log
  dbfilename dump-6379.rdb
  daemonize yes
  databases 1
  masterauth "demo"
  requirepass "demo"
  #### Master server close backsave ####
  save ""
  stop-writes-on-bgsave-error no

server [备] 配置

vim ${CODIS_HOME}/config/server-slave-6380.conf
  bind 0.0.0.0
  port 6380
  dir /opt/gopath/src/github.com/CodisLabs/codis/data
  pidfile ../logs/redis-6380.pid
  logfile ../logs/redis-6380.log
  dbfilename dump-6380.rdb
  daemonize yes
  databases 1
  masterauth "demo"
  requirepass "demo"
  #### rdb config ####
  save 900 1   
  save 300 10
  save 60 10000
  stop-writes-on-bgsave-error no
  slaveof 127.0.0.1 6379
  slave-read-only yes

server 启动脚本 [主 / 备]

#codis-server [主]启动脚本
vim /opt/codis/server-master-start.sh
  codis-server ${CODIS_HOME}/config/server-master-6379.conf

#codis-server [主]关闭脚本
vim /opt/codis/server-master-stop.sh
  redis-cli -p 6379 <<EOF
  auth demo 
  shutdown
  EOF

#codis-server [备]启动脚本
vim /opt/codis/server-slave-start.sh
  codis-server ${CODIS_HOME}/config/server-slave-6380.conf

#codos-server [备]关闭脚本
vim /opt/codis/server-slave-stop.sh
  redis-cli -p 6380 <<EOF
  auth demo 
  shutdown
  EOF

fe 配置

cd ${CODIS_HOME}/config
vim codis-1234.json
  [
      {
          "name": "codis-demo",
          "dashboard": "127.0.0.1:18080"
      }
  ]

fe 启动脚本

vim /opt/codis/fe-start.sh
  PORT=1234
  NCPU=4
  ASSETS=${CODIS_HOME}/bin/assets
  CONFIG=${CODIS_HOME}/config/codis-${PORT}.json 
  LOG=${CODIS_HOME}/logs/fe-${PORT}.log
  PID=${CODIS_HOME}/logs/fe-${PORT}.pid
  LEVEL="WARN"
  codis-fe --ncpu=${NCPU} --assets-dir=${ASSETS} --log=${LOG} --pidfile=${PID} --log-level=${LEVEL} --dashboard-list=${CONFIG} --listen=0.0.0.0:${PORT} &

fe 关闭脚本

vim /opt/codis/fe-stop.sh
  PORT=1234
  PID=${CODIS_HOME}/logs/fe-${PORT}.pid
  cat ${PID} | xargs kill

启动 codis 所有相关进程

cd /opt/zookeeper/zookeeper-3.4.14/bin
./zkServer.sh start

/opt/codis/dashboard-stop.sh
/opt/codis/dashboard-start.sh

/opt/codis/server-master-stop.sh
/opt/codis/server-master-start.sh
/opt/codis/server-slave-stop.sh
/opt/codis/server-slave-start.sh

/opt/codis/proxy-stop.sh
/opt/codis/proxy-start.sh

/opt/codis/fe-stop.sh
/opt/codis/fe-start.sh

/opt/codis/ha-start.sh

验证 fee 进程

#验证,能访问初步说明正常启动了
curl http://localhost:1234/

3、配置 Codis 集群

codis 集群配置

将组件关联到集群中,本文将通过 codis-admin 命令配置。除了 codis-admin,还可以通过 fe 界面(http://localhost:1234/)配置。

#添加代理
#注:create-proxy -x <内网ip>:11080
codis-admin --dashboard=127.0.0.1:18080 --create-proxy -x localhost:11080

#添加组
codis-admin --dashboard=127.0.0.1:18080 --create-group --gid=1

#给组添加/删除Redis服务【group-addr】【group-del:删除时不带datacenter参数】,每个主与备为一组
codis-admin --dashboard=127.0.0.1:18080 --group-add --gid=1 --datacenter=center1 --addr=localhost:6379
codis-admin --dashboard=127.0.0.1:18080 --group-add --gid=1 --datacenter=center1 --addr=localhost:6380

#初始化slots并设置server group服务的slot 范围
codis-admin --dashboard=127.0.0.1:18080 --slot-action --create-range --beg=0 --end=1023 --gid=1

通过 jodis 访问 codis 示例

#所有工作都已经准备完成,接下来就是将编译好的codis分发到各个主机上
#jodis客户端连接
public static final String zkAddr = "localhost:2181";
public static final int zkSessionTimeoutMs = 30000;
public static final String zkProxyDir = "/jodis/codis-demo";
public static void main(String[] args) {
    JedisResourcePool jedisPool = RoundRobinJedisPool.create().curatorClient(zkAddr, zkSessionTimeoutMs).password("codis-demo")
            .zkProxyDir(zkProxyDir).build();
    for (int i = 0; i < 1024; i++) {
        try (Jedis jedis = jedisPool.getResource()) {
            jedis.set("test:" + i, "bar" + i);
            String value = jedis.get("test:" + i);
            System.out.println(value);
        }
    }
}