blob: b933158a54827a0db1432e147910eccaef1706ea (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
|
{ config, lib, pkgs, ... }: let
cfg = config.lass.drbd;
slib = import <stockholm/lib>;
in {
options = {
lass.drbd = lib.mkOption {
default = {};
type = lib.types.attrsOf (lib.types.submodule ({ config, ... }: {
options = {
name = lib.mkOption {
type = lib.types.str;
default = config._module.args.name;
};
blockMinor = lib.mkOption {
type = lib.types.int;
default = lib.mod (slib.genid config.name) 16000; # TODO get max_id fron drbd
};
port = lib.mkOption {
type = lib.types.int;
default = 20000 + config.blockMinor;
};
peers = lib.mkOption {
type = lib.types.listOf slib.types.host;
};
disk = lib.mkOption {
type = lib.types.str;
default = "/dev/loop${toString config.blockMinor}";
};
drbdConfig = lib.mkOption {
type = lib.types.path;
internal = true;
default = pkgs.writeText "drbd-${config.name}.conf" ''
resource ${config.name} {
net {
protocol a;
ping-int 10;
csums-alg crc32c;
connect-int 3;
after-sb-0pri discard-older-primary;
after-sb-1pri discard-secondary;
# seems to be drbd-proxy premium feature
on-congestion pull-ahead;
congestion-fill 1G;
congestion-extents 500;
sndbuf-size 10M;
max-epoch-size 20000;
}
device minor ${toString config.blockMinor};
disk ${config.disk};
meta-disk internal;
${slib.indent (lib.concatStrings (lib.imap1 (i: peer: /* shell */ ''
on ${peer.name} {
address ${peer.nets.retiolum.ip4.addr}:${toString config.port};
node-id ${toString i};
}
'') config.peers))}
connection-mesh {
hosts ${lib.concatMapStringsSep " " (peer: peer.name) config.peers};
}
}
'';
};
};
}));
};
};
config = lib.mkIf (cfg != {}) {
boot.extraModulePackages = [
(pkgs.linuxPackages.callPackage ../5pkgs/drbd9/default.nix {})
];
boot.extraModprobeConfig = ''
options drbd usermode_helper=/run/current-system/sw/bin/drbdadm
'';
services.udev.packages = [ pkgs.drbd ];
boot.kernelModules = [ "drbd" ];
environment.systemPackages = [
pkgs.drbd
(pkgs.writers.writeDashBin "drbd-change-nodeid" ''
# https://linbit.com/drbd-user-guide/drbd-guide-9_0-en/#s-using-truck-based-replication
set -efux
if [ "$#" -ne 2 ]; then
echo '$1 needs to be drbd volume name'
echo '$2 needs to be new node id'
exit 1
fi
TMPDIR=$(mktemp -d)
trap 'rm -rf $TMPDIR' EXIT
V=$1
NODE_TO=$2
META_DATA_LOCATION=internal
${pkgs.drbd}/bin/drbdadm -- --force dump-md $V > "$TMPDIR"/md_orig.txt
NODE_FROM=$(cat "$TMPDIR"/md_orig.txt | ${pkgs.gnused}/bin/sed -n 's/^node-id \(.*\);$/\1/p')
${pkgs.gnused}/bin/sed -e "s/node-id $NODE_FROM/node-id $NODE_TO/" \
-e "s/^peer.$NODE_FROM. /peer-NEW /" \
-e "s/^peer.$NODE_TO. /peer[$NODE_FROM] /" \
-e "s/^peer-NEW /peer[$NODE_TO] /" \
< "$TMPDIR"/md_orig.txt > "$TMPDIR"/md.txt
drbdmeta --force $(drbdadm sh-minor $V) v09 $(drbdadm sh-md-dev $V) $META_DATA_LOCATION restore-md "$TMPDIR"/md.txt
'')
];
networking.firewall.allowedTCPPorts = map (device: device.port) (lib.attrValues cfg);
systemd.services = lib.mapAttrs' (_: device:
lib.nameValuePair "drbd-${device.name}" {
after = [ "systemd-udev.settle.service" "network.target" "retiolum.service" ];
wants = [ "systemd-udev.settle.service" ];
wantedBy = [ "multi-user.target" ];
serviceConfig = {
RemainAfterExit = true;
ExecStart = pkgs.writers.writeDash "start-drbd-${device.name}" ''
set -efux
mkdir -p /var/lib/sync-containers2
${lib.optionalString (device.disk == "/dev/loop${toString device.blockMinor}") ''
if ! test -e /var/lib/sync-containers2/${device.name}.disk; then
truncate -s 10G /var/lib/sync-containers2/${device.name}.disk
fi
if ! ${pkgs.util-linux}/bin/losetup /dev/loop${toString device.blockMinor}; then
${pkgs.util-linux}/bin/losetup /dev/loop${toString device.blockMinor} /var/lib/sync-containers2/${device.name}.disk
fi
''}
if ! ${pkgs.drbd}/bin/drbdadm adjust ${device.name}; then
${pkgs.drbd}/bin/drbdadm down ${device.name}
${pkgs.drbd}/bin/drbdadm create-md ${device.name}/0 --max-peers 31
${pkgs.drbd}/bin/drbdadm up ${device.name}
fi
'';
ExecStop = pkgs.writers.writeDash "stop-drbd-${device.name}" ''
set -efux
${pkgs.drbd}/bin/drbdadm -c ${device.drbdConfig} down ${device.name}
${lib.optionalString (device.disk == "/dev/loop${toString device.blockMinor}") ''
${pkgs.util-linux}/bin/losetup -d /dev/loop${toString device.blockMinor}
''}
'';
};
}
) cfg;
environment.etc."drbd.conf".text = ''
global {
usage-count yes;
}
${lib.concatMapStrings (device: /* shell */ ''
include ${device.drbdConfig};
'') (lib.attrValues cfg)}
'';
};
}
|
