Skip to content

Commit d44695b

Browse files
authored
Merge pull request #900 from lizhuqi/guest-config
add nvme script and rules as static files
2 parents db9e12f + 55a9859 commit d44695b

File tree

5 files changed

+279
-0
lines changed

5 files changed

+279
-0
lines changed

Dockerfile

+3
Original file line numberDiff line numberDiff line change
@@ -63,4 +63,7 @@ COPY --from=debian /lib/x86_64-linux-gnu/libselinux.so.1 /lib/x86_64-linux-gnu/l
6363
COPY --from=debian /lib/x86_64-linux-gnu/libtinfo.so.6 /lib/x86_64-linux-gnu/libtinfo.so.6
6464
COPY --from=debian /lib/x86_64-linux-gnu/libuuid.so.1 /lib/x86_64-linux-gnu/libuuid.so.1
6565

66+
# Copy NVME support required script and rules into distroless base.
67+
COPY deploy/kubernetes/udev/google_nvme_id /lib/udev_containerized/google_nvme_id
68+
6669
ENTRYPOINT ["/gce-pd-csi-driver"]

deploy/kubernetes/README.md

+4
Original file line numberDiff line numberDiff line change
@@ -23,3 +23,7 @@ The current structure for kustomization is as follows. Note that Windows support
2323
* `prow-gke-release-staging-rc-{k8s-minor}`: Used for prow tests. Contains deployment specs of a driver for given k8s minor version release.
2424
* `prow-gke-release-staging-rc-head`: Used for prow tests. Contains deployment specs of a driver with latest sidecar images, for latest k8s master.
2525
* `stable`, `prow-gke-release-staging-rc`: Soon to be removed!
26+
27+
## NVME support
28+
udev folder under deploy/kubernetes contains google_nvme_id script required for NVME support. Source is downloaded from [GoogleCloudPlatform/guest-configs](https://github.com/GoogleCloudPlatform/guest-configs). README file in the folder contains the downloaded version.
29+
Execute deploy/kubernetes/update-nvme.sh to update. See releases available in [GoogleCloudPlatform/guest-configs/releases](https://github.com/GoogleCloudPlatform/guest-configs/releases).

deploy/kubernetes/udev/README

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
GUEST_CONFIGS_VERSION: 20211116.00

deploy/kubernetes/udev/google_nvme_id

+245
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,245 @@
1+
#!/bin/bash
2+
# Copyright 2020 Google Inc. All Rights Reserved.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
# Used to generate symlinks for PD-NVMe devices using the disk names reported by
17+
# the metadata server
18+
19+
# Locations of the script's dependencies
20+
readonly nvme_cli_bin=/usr/sbin/nvme
21+
22+
# Bash regex to parse device paths and controller identification
23+
readonly NAMESPACE_NUMBER_REGEX="/dev/nvme[[:digit:]]+n([[:digit:]]+).*"
24+
readonly PARTITION_NUMBER_REGEX="/dev/nvme[[:digit:]]+n[[:digit:]]+p([[:digit:]]+)"
25+
readonly PD_NVME_REGEX="sn[[:space:]]+:[[:space]]+nvme_card-pd"
26+
27+
# Globals used to generate the symlinks for a PD-NVMe disk. These are populated
28+
# by the identify_pd_disk function and exported for consumption by udev rules.
29+
ID_SERIAL=''
30+
ID_SERIAL_SHORT=''
31+
32+
#######################################
33+
# Helper function to log an error message to stderr.
34+
# Globals:
35+
# None
36+
# Arguments:
37+
# String to print as the log message
38+
# Outputs:
39+
# Writes error to STDERR
40+
#######################################
41+
function err() {
42+
echo "[$(date +'%Y-%m-%dT%H:%M:%S%z')]: $*" >&2
43+
}
44+
45+
#######################################
46+
# Retrieves the device name for an NVMe namespace using nvme-cli.
47+
# Globals:
48+
# Uses nvme_cli_bin
49+
# Arguments:
50+
# The path to the nvme namespace (/dev/nvme0n?)
51+
# Outputs:
52+
# The device name parsed from the JSON in the vendor ext of the ns-id command.
53+
# Returns:
54+
# 0 if the device name for the namespace could be retrieved, 1 otherwise
55+
#######################################
56+
function get_namespace_device_name() {
57+
local nvme_json
58+
nvme_json="$("$nvme_cli_bin" id-ns -b "$1" | xxd -p -seek 384 | xxd -p -r)"
59+
if [[ $? -ne 0 ]]; then
60+
return 1
61+
fi
62+
63+
if [[ -z "$nvme_json" ]]; then
64+
err "NVMe Vendor Extension disk information not present"
65+
return 1
66+
fi
67+
68+
local device_name
69+
device_name="$(echo "$nvme_json" | grep device_name | sed -e 's/.*"device_name":[ \t]*"\([a-zA-Z0-9_-]\+\)".*/\1/')"
70+
71+
# Error if our device name is empty
72+
if [[ -z "$device_name" ]]; then
73+
err "Empty name"
74+
return 1
75+
fi
76+
77+
echo "$device_name"
78+
return 0
79+
}
80+
81+
#######################################
82+
# Retrieves the nsid for an NVMe namespace
83+
# Globals:
84+
# None
85+
# Arguments:
86+
# The path to the nvme namespace (/dev/nvme0n*)
87+
# Outputs:
88+
# The namespace number/id
89+
# Returns:
90+
# 0 if the namespace id could be retrieved, 1 otherwise
91+
#######################################
92+
function get_namespace_number() {
93+
local dev_path="$1"
94+
local namespace_number
95+
if [[ "$dev_path" =~ $NAMESPACE_NUMBER_REGEX ]]; then
96+
namespace_number="${BASH_REMATCH[1]}"
97+
else
98+
return 1
99+
fi
100+
101+
echo "$namespace_number"
102+
return 0
103+
}
104+
105+
#######################################
106+
# Retrieves the partition number for a device path if it exists
107+
# Globals:
108+
# None
109+
# Arguments:
110+
# The path to the device partition (/dev/nvme0n*p*)
111+
# Outputs:
112+
# The value after 'p' in the device path, or an empty string if the path has
113+
# no partition.
114+
#######################################
115+
function get_partition_number() {
116+
local dev_path="$1"
117+
local partition_number
118+
if [[ "$dev_path" =~ $PARTITION_NUMBER_REGEX ]]; then
119+
partition_number="${BASH_REMATCH[1]}"
120+
echo "$partition_number"
121+
else
122+
echo ''
123+
fi
124+
return 0
125+
}
126+
127+
#######################################
128+
# Generates a symlink for a PD-NVMe device using the metadata's disk name.
129+
# Primarily used for testing but can be used if the script is directly invoked.
130+
# Globals:
131+
# Uses ID_SERIAL_SHORT (can be populated by identify_pd_disk)
132+
# Arguments:
133+
# The device path for the disk
134+
#######################################
135+
function gen_symlink() {
136+
local dev_path="$1"
137+
local partition_number="$(get_partition_number "$dev_path")"
138+
139+
if [[ -n "$partition_number" ]]; then
140+
ln -s "$dev_path" /dev/disk/by-id/google-"$ID_SERIAL_SHORT"-part"$partition_number" > /dev/null 2>&1
141+
else
142+
ln -s "$dev_path" /dev/disk/by-id/google-"$ID_SERIAL_SHORT" > /dev/null 2>&1
143+
fi
144+
145+
return 0
146+
}
147+
148+
#######################################
149+
# Populates the ID_* global variables with a disk's device name and namespace
150+
# Globals:
151+
# Populates ID_SERIAL_SHORT, and ID_SERIAL
152+
# Arguments:
153+
# The device path for the disk
154+
# Returns:
155+
# 0 on success and 1 if an error occurrs
156+
#######################################
157+
function identify_pd_disk() {
158+
local dev_path="$1"
159+
local dev_name
160+
dev_name="$(get_namespace_device_name "$dev_path")"
161+
if [[ $? -ne 0 ]]; then
162+
return 1
163+
fi
164+
165+
ID_SERIAL_SHORT="$dev_name"
166+
ID_SERIAL="Google_PersistentDisk_${ID_SERIAL_SHORT}"
167+
return 0
168+
}
169+
170+
function print_help_message() {
171+
echo "Usage: google_nvme_id [-s] [-h] -d device_path"
172+
echo " -d <device_path> (Required): Specifies the path to generate a name"
173+
echo " for. This needs to be a path to an nvme device or namespace"
174+
echo " -s: Create symbolic link for the disk under /dev/disk/by-id."
175+
echo " Otherwise, the disk name will be printed to STDOUT"
176+
echo " -h: Print this help message"
177+
}
178+
179+
function main() {
180+
local opt_gen_symlink='false'
181+
local device_path=''
182+
183+
while getopts :d:sh flag; do
184+
case "$flag" in
185+
d) device_path="$OPTARG";;
186+
s) opt_gen_symlink='true';;
187+
h) print_help_message
188+
return 0
189+
;;
190+
:) echo "Invalid option: ${OPTARG} requires an argument" 1>&2
191+
return 1
192+
;;
193+
*) return 1
194+
esac
195+
done
196+
197+
if [[ -z "$device_path" ]]; then
198+
echo "Device path (-d) argument required. Use -h for full usage." 1>&2
199+
exit 1
200+
fi
201+
202+
# Ensure the nvme-cli command is installed
203+
command -v "$nvme_cli_bin" > /dev/null 2>&1
204+
if [[ $? -ne 0 ]]; then
205+
err "The nvme utility (/usr/sbin/nvme) was not found. You may need to run \
206+
with sudo or install nvme-cli."
207+
return 1
208+
fi
209+
210+
# Ensure the passed device is actually an NVMe device
211+
"$nvme_cli_bin" id-ctrl "$device_path" &>/dev/null
212+
if [[ $? -ne 0 ]]; then
213+
err "Passed device was not an NVMe device. (You may need to run this \
214+
script as root/with sudo)."
215+
return 1
216+
fi
217+
218+
# Detect the type of attached nvme device
219+
local controller_id
220+
controller_id=$("$nvme_cli_bin" id-ctrl "$device_path")
221+
if [[ ! "$controller_id" =~ nvme_card-pd ]] ; then
222+
err "Device is not a PD-NVMe device"
223+
return 1
224+
fi
225+
226+
# Fill the global variables for the id command for the given disk type
227+
# Error messages will be printed closer to error, no need to reprint here
228+
identify_pd_disk "$device_path"
229+
if [[ $? -ne 0 ]]; then
230+
return $?
231+
fi
232+
233+
# Gen symlinks or print out the globals set by the identify command
234+
if [[ "$opt_gen_symlink" == 'true' ]]; then
235+
gen_symlink "$device_path"
236+
else
237+
# These will be consumed by udev
238+
echo "ID_SERIAL_SHORT=${ID_SERIAL_SHORT}"
239+
echo "ID_SERIAL=${ID_SERIAL}"
240+
fi
241+
242+
return $?
243+
244+
}
245+
main "$@"

deploy/kubernetes/update-nvme.sh

+26
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
#!/bin/bash
2+
3+
# This scripts is to download source of a release from GoogleCloudPlatform/guest-configs repo
4+
# and copy script and rules required for NVMe support to be under deploy/kubernetes
5+
#
6+
# Args:
7+
# GUEST_CONFIGS_VERSION: The version of the guest configs release
8+
9+
set -o nounset
10+
set -o errexit
11+
12+
readonly PKGDIR="${GOPATH}/src/sigs.k8s.io/gcp-compute-persistent-disk-csi-driver"
13+
readonly INSTALL_DIR="${PKGDIR}/tmp"
14+
readonly DEST="${PKGDIR}/deploy/kubernetes/udev"
15+
readonly SOURCE_FOLDER_NAME="guest-configs-${GUEST_CONFIGS_VERSION}"
16+
17+
function cleanup {
18+
rm -rf "$INSTALL_DIR"
19+
}
20+
21+
trap cleanup EXIT
22+
23+
curl --create-dirs -O --output-dir ${INSTALL_DIR} -L https://github.com/GoogleCloudPlatform/guest-configs/archive/refs/tags/${GUEST_CONFIGS_VERSION}.tar.gz
24+
tar -xzf ${INSTALL_DIR}/${GUEST_CONFIGS_VERSION}.tar.gz -C ${INSTALL_DIR}
25+
cp -R ${INSTALL_DIR}/${SOURCE_FOLDER_NAME}/src/lib/udev/google_nvme_id ${DEST}
26+
echo "GUEST_CONFIGS_VERSION: " ${GUEST_CONFIGS_VERSION} > ${DEST}/README

0 commit comments

Comments
 (0)