diff --git a/bin/make-test-certs.sh b/bin/make-test-certs.sh new file mode 100755 index 000000000..d7a90fed0 --- /dev/null +++ b/bin/make-test-certs.sh @@ -0,0 +1,119 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Create test TLS certificates for use with rclone. + +OUT_DIR="${OUT_DIR:-./tls-test}" +CA_SUBJ="${CA_SUBJ:-/C=US/ST=Test/L=Test/O=Test Org/OU=Test Unit/CN=Test Root CA}" +SERVER_CN="${SERVER_CN:-localhost}" +CLIENT_CN="${CLIENT_CN:-Test Client}" +CLIENT_KEY_PASS="${CLIENT_KEY_PASS:-testpassword}" + +CA_DAYS=${CA_DAYS:-3650} +SERVER_DAYS=${SERVER_DAYS:-825} +CLIENT_DAYS=${CLIENT_DAYS:-825} + +mkdir -p "$OUT_DIR" +cd "$OUT_DIR" + +# Create OpenSSL config + +# CA extensions +cat > ca_openssl.cnf <<'EOF' +[ ca_ext ] +basicConstraints = critical, CA:true, pathlen:1 +keyUsage = critical, keyCertSign, cRLSign +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid:always,issuer +EOF + +# Server extensions (SAN includes localhost + loopback IP) +cat > server_openssl.cnf < client_openssl.cnf <<'EOF' +[ client_ext ] +basicConstraints = critical, CA:false +keyUsage = critical, digitalSignature, keyEncipherment +extendedKeyUsage = clientAuth +subjectKeyIdentifier = hash +authorityKeyIdentifier = keyid,issuer +EOF + +echo "Create CA key, CSR, and self-signed CA cert" +if [ ! -f ca.key.pem ]; then + openssl genrsa -out ca.key.pem 4096 + chmod 600 ca.key.pem +fi + +openssl req -new -key ca.key.pem -subj "$CA_SUBJ" -out ca.csr.pem + +openssl x509 -req -in ca.csr.pem -signkey ca.key.pem \ + -sha256 -days "$CA_DAYS" \ + -extfile ca_openssl.cnf -extensions ca_ext \ + -out ca.cert.pem + +echo "Create server key (NO PASSWORD) and cert signed by CA" +openssl genrsa -out server.key.pem 2048 +chmod 600 server.key.pem + +openssl req -new -key server.key.pem -subj "/CN=${SERVER_CN}" -out server.csr.pem + +openssl x509 -req -in server.csr.pem \ + -CA ca.cert.pem -CAkey ca.key.pem -CAcreateserial \ + -out server.cert.pem -days "$SERVER_DAYS" -sha256 \ + -extfile server_openssl.cnf -extensions server_ext + +echo "Create client key (PASSWORD-PROTECTED), CSR, and cert" +openssl genrsa -aes256 -passout pass:"$CLIENT_KEY_PASS" -out client.key.pem 2048 +chmod 600 client.key.pem + +openssl req -new -key client.key.pem -passin pass:"$CLIENT_KEY_PASS" \ + -subj "/CN=${CLIENT_CN}" -out client.csr.pem + +openssl x509 -req -in client.csr.pem \ + -CA ca.cert.pem -CAkey ca.key.pem -CAcreateserial \ + -out client.cert.pem -days "$CLIENT_DAYS" -sha256 \ + -extfile client_openssl.cnf -extensions client_ext + +echo "Verify chain" +openssl verify -CAfile ca.cert.pem server.cert.pem client.cert.pem + +echo "Done" + +echo +echo "Summary" +echo "-------" +printf "%-22s %s\n" \ + "CA key:" "ca.key.pem" \ + "CA cert:" "ca.cert.pem" \ + "Server key:" "server.key.pem (no password)" \ + "Server CSR:" "server.csr.pem" \ + "Server cert:" "server.cert.pem (SAN: ${SERVER_CN}, 127.0.0.1)" \ + "Client key:" "client.key.pem (encrypted)" \ + "Client CSR:" "client.csr.pem" \ + "Client cert:" "client.cert.pem" \ + "Client key password:" "$CLIENT_KEY_PASS" + +echo +echo "Test rclone server" +echo +echo "rclone serve http -vv --addr :8080 --cert ${OUT_DIR}/server.cert.pem --key ${OUT_DIR}/server.key.pem --client-ca ${OUT_DIR}/ca.cert.pem ." + +echo +echo "Test rclone client" +echo +echo "rclone lsf :http: --http-url 'https://localhost:8080' --ca-cert ${OUT_DIR}/ca.cert.pem --client-cert ${OUT_DIR}/client.cert.pem --client-key ${OUT_DIR}/client.key.pem --client-pass \$(rclone obscure $CLIENT_KEY_PASS)" +echo diff --git a/docs/content/docs.md b/docs/content/docs.md index ba9435781..80ee20042 100644 --- a/docs/content/docs.md +++ b/docs/content/docs.md @@ -3000,6 +3000,20 @@ The `--client-key` flag is required too when using this. This loads the PEM encoded client side private key used for mutual TLS authentication. Used in conjunction with `--client-cert`. +Supported types are: + +- Unencrypted PKCS#1 ("BEGIN RSA PRIVATE KEY") +- Unencrypted PKCS#8 ("BEGIN PRIVATE KEY") +- Encrypted PKCS#8 ("BEGIN ENCRYPTED PRIVATE KEY") +- Legacy PEM encryption (e.g., DEK-Info headers), which are automatically detected. + +### --client-pass string + +This can be used to supply an optional password to decrypt the client key file. + +**NB** the password should be obscured so it should be the output of +`rclone obscure YOURPASSWORD`. + ### --no-check-certificate `--no-check-certificate` controls whether a client verifies the diff --git a/fs/config.go b/fs/config.go index 04a18aafe..ce0a1df9e 100644 --- a/fs/config.go +++ b/fs/config.go @@ -438,6 +438,12 @@ var ConfigOptionsInfo = Options{{ Default: "", Help: "Client SSL private key (PEM) for mutual TLS auth", Groups: "Networking", +}, { + Name: "client_pass", + Default: "", + Help: "Password for client SSL private key (PEM) for mutual TLS auth (obscured)", + Groups: "Networking", + IsPassword: true, }, { Name: "multi_thread_cutoff", Default: SizeSuffix(256 * 1024 * 1024), @@ -644,6 +650,7 @@ type ConfigInfo struct { CaCert []string `config:"ca_cert"` // Client Side CA ClientCert string `config:"client_cert"` // Client Side Cert ClientKey string `config:"client_key"` // Client Side Key + ClientPass string `config:"client_pass"` // Client Side Key Password (obscured) MultiThreadCutoff SizeSuffix `config:"multi_thread_cutoff"` MultiThreadStreams int `config:"multi_thread_streams"` MultiThreadSet bool `config:"multi_thread_set"` // whether MultiThreadStreams was set (set in fs/config/configflags) diff --git a/fs/fshttp/http.go b/fs/fshttp/http.go index de9ca3cf8..65684f3db 100644 --- a/fs/fshttp/http.go +++ b/fs/fshttp/http.go @@ -6,6 +6,8 @@ import ( "context" "crypto/tls" "crypto/x509" + "encoding/pem" + "errors" "fmt" "net" "net/http" @@ -18,7 +20,9 @@ import ( "github.com/rclone/rclone/fs" "github.com/rclone/rclone/fs/accounting" + "github.com/rclone/rclone/fs/config/obscure" "github.com/rclone/rclone/lib/structs" + "github.com/youmark/pkcs8" "golang.org/x/net/publicsuffix" ) @@ -48,6 +52,156 @@ func ResetTransport() { noTransport = new(sync.Once) } +// LoadKeyPair loads a TLS certificate and private key from PEM-encoded files, +// with extended support for encrypted private keys. +// +// This function is designed as a robust replacement for tls.X509KeyPair, +// providing the same core functionality but adding support for +// password-protected private keys. +// +// The certificate file (certFile) must contain one or more PEM-encoded +// certificates. The first certificate is treated as the leaf certificate, and +// any subsequent certificates are treated as its chain. +// +// The key file (keyFile) must contain a PEM-encoded private key. Supported +// formats are: +// +// - Unencrypted PKCS#1 ("BEGIN RSA PRIVATE KEY") +// - Unencrypted PKCS#8 ("BEGIN PRIVATE KEY") +// - Encrypted PKCS#8 ("BEGIN ENCRYPTED PRIVATE KEY") +// - Legacy PEM encryption (e.g., DEK-Info headers), which are automatically detected. +// +// The password parameter is used to decrypt the private key. If the +// key is not encrypted, this parameter is ignored and can be an empty +// string. The password should be an obscured string. +// +// On success, it returns a fully populated tls.Certificate struct, including the +// Leaf certificate field. +func LoadKeyPair(certFile, keyFile, password string) (cert tls.Certificate, err error) { + certPEM, err := os.ReadFile(certFile) + if err != nil { + return cert, fmt.Errorf("read cert: %w", err) + } + keyPEM, err := os.ReadFile(keyFile) + if err != nil { + return cert, fmt.Errorf("read key: %w", err) + } + if password != "" { + password, err = obscure.Reveal(password) + if err != nil { + return cert, fmt.Errorf("reveal key password: %w", err) + } + } + + // Fast path: unencrypted PKCS#1/PKCS#8 + cert, err = tls.X509KeyPair(certPEM, keyPEM) + if err == nil { + if len(cert.Certificate) == 0 { + return cert, errors.New("no certificates parsed") + } + leaf, err := x509.ParseCertificate(cert.Certificate[0]) + if err != nil { + return cert, fmt.Errorf("parse leaf: %w", err) + } + cert.Leaf = leaf + return cert, nil + } + + // Decrypt / parse key manually + block, rest := pem.Decode(keyPEM) + if block == nil { + return cert, errors.New("no PEM block in key") + } + if len(rest) != 0 { + fs.Debugf(nil, "Trailing data (%d bytes) in key PEM loaded from %q", len(rest), keyFile) + } + + var privKey any + switch { + case block.Type == "ENCRYPTED PRIVATE KEY": + if password == "" { + return cert, errors.New("key is encrypted but no --client-pass provided") + } + privKey, err = pkcs8.ParsePKCS8PrivateKey(block.Bytes, []byte(password)) + if err != nil { + return cert, fmt.Errorf("parse encrypted PKCS#8: %w", err) + } + + case x509.IsEncryptedPEMBlock(block): //nolint:staticcheck // this is Legacy and insecure + if password == "" { + return cert, errors.New("key is encrypted but no --client-pass provided") + } + der, err := x509.DecryptPEMBlock(block, []byte(password)) //nolint:staticcheck // this is Legacy and insecure + if err != nil { + return cert, fmt.Errorf("decrypt PEM key: %w", err) + } + // Try PKCS#8, then RSA PKCS#1, then EC + if k, kerr1 := x509.ParsePKCS8PrivateKey(der); kerr1 == nil { + privKey = k + } else if k, kerr2 := x509.ParsePKCS1PrivateKey(der); kerr2 == nil { + privKey = k + } else if k, kerr3 := x509.ParseECPrivateKey(der); kerr3 == nil { + privKey = k + } else { + return cert, fmt.Errorf("parse decrypted key: pkcs8: %v, pkcs1: %v, ec: %v", kerr1, kerr2, kerr3) + } + + default: + // Unencrypted specific types + switch block.Type { + case "PRIVATE KEY": + k, kerr := x509.ParsePKCS8PrivateKey(block.Bytes) + if kerr != nil { + return cert, fmt.Errorf("parse PKCS#8: %w", kerr) + } + privKey = k + case "RSA PRIVATE KEY": + k, kerr := x509.ParsePKCS1PrivateKey(block.Bytes) + if kerr != nil { + return cert, fmt.Errorf("parse PKCS#1 RSA: %w", kerr) + } + privKey = k + case "EC PRIVATE KEY": + k, kerr := x509.ParseECPrivateKey(block.Bytes) + if kerr != nil { + return cert, fmt.Errorf("parse EC: %w", kerr) + } + privKey = k + default: + return cert, fmt.Errorf("unsupported key type %q", block.Type) + } + } + + // Build cert chain from PEM + var certDERs [][]byte + for rest := certPEM; ; { + var b *pem.Block + b, rest = pem.Decode(rest) + if b == nil { + break + } + if b.Type == "CERTIFICATE" { + certDERs = append(certDERs, b.Bytes) + } + } + if len(certDERs) == 0 { + return cert, fmt.Errorf("no CERTIFICATE blocks in %s", certFile) + } + + cert = tls.Certificate{ + Certificate: certDERs, + PrivateKey: privKey, + } + + // Leaf is always the first certificate + cert.Leaf, err = x509.ParseCertificate(cert.Certificate[0]) + if err != nil { + return cert, fmt.Errorf("parse leaf: %w", err) + } + + return cert, nil +} + // NewTransportCustom returns an http.RoundTripper with the correct timeouts. // The customize function is called if set to give the caller an opportunity to // customize any defaults in the Transport. @@ -85,17 +239,10 @@ func NewTransportCustom(ctx context.Context, customize func(*http.Transport)) *T if ci.ClientCert == "" || ci.ClientKey == "" { fs.Fatalf(nil, "Both --client-cert and --client-key must be set") } - cert, err := tls.LoadX509KeyPair(ci.ClientCert, ci.ClientKey) + cert, err := LoadKeyPair(ci.ClientCert, ci.ClientKey, ci.ClientPass) if err != nil { fs.Fatalf(nil, "Failed to load --client-cert/--client-key pair: %v", err) } - if cert.Leaf == nil { - // Leaf is always the first certificate - cert.Leaf, err = x509.ParseCertificate(cert.Certificate[0]) - if err != nil { - fs.Fatalf(nil, "Failed to parse the certificate") - } - } t.TLSClientConfig.Certificates = []tls.Certificate{cert} } @@ -187,14 +334,12 @@ func NewClientWithUnixSocket(ctx context.Context, path string) *http.Client { // * Updates metrics type Transport struct { *http.Transport + ci *fs.ConfigInfo dump fs.DumpFlags filterRequest func(req *http.Request) userAgent string headers []*fs.HTTPOption metrics *Metrics - // Filename of the client cert in case we need to reload it - clientCert string - clientKey string // Mutex for serializing attempts at reloading the certificates reloadMutex sync.Mutex } @@ -203,13 +348,12 @@ type Transport struct { // roundtrips including the body if logBody is set. func newTransport(ci *fs.ConfigInfo, transport *http.Transport) *Transport { return &Transport{ - Transport: transport, - dump: ci.Dump, - userAgent: ci.UserAgent, - headers: ci.Headers, - metrics: DefaultMetrics, - clientCert: ci.ClientCert, - clientKey: ci.ClientKey, + Transport: transport, + ci: ci, + dump: ci.Dump, + userAgent: ci.UserAgent, + headers: ci.Headers, + metrics: DefaultMetrics, } } @@ -309,20 +453,10 @@ func (t *Transport) reloadCertificates() { if !isCertificateExpired(t.TLSClientConfig) { return } - - cert, err := tls.LoadX509KeyPair(t.clientCert, t.clientKey) + cert, err := LoadKeyPair(t.ci.ClientCert, t.ci.ClientKey, t.ci.ClientPass) if err != nil { fs.Fatalf(nil, "Failed to load --client-cert/--client-key pair: %v", err) } - // Check if we need to parse the certificate again, we need it - // for checking the expiration date - if cert.Leaf == nil { - // Leaf is always the first certificate - cert.Leaf, err = x509.ParseCertificate(cert.Certificate[0]) - if err != nil { - fs.Fatalf(nil, "Failed to parse the certificate") - } - } t.TLSClientConfig.Certificates = []tls.Certificate{cert} }