diff --git a/.travis.yml b/.travis.yml index 8d4428d1f..43cc9c2b6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,27 +2,23 @@ language: go go_import_path: github.com/globalsign/mgo +go: + - 1.8.x + - 1.9.x + env: global: - BUCKET=https://s3.eu-west-2.amazonaws.com/globalsign-mgo + - FASTDL=https://fastdl.mongodb.org/linux matrix: - - GO=1.7 MONGODB=x86_64-2.6.11 - - GO=1.8.x MONGODB=x86_64-2.6.11 - - GO=1.7 MONGODB=x86_64-3.0.9 - - GO=1.8.x MONGODB=x86_64-3.0.9 - - GO=1.7 MONGODB=x86_64-3.2.3-nojournal - - GO=1.8.x MONGODB=x86_64-3.2.3-nojournal - - GO=1.7 MONGODB=x86_64-3.2.12 - - GO=1.8.x MONGODB=x86_64-3.2.12 - - GO=1.7 MONGODB=x86_64-3.2.16 - - GO=1.8.x MONGODB=x86_64-3.2.16 - - GO=1.7 MONGODB=x86_64-3.4.8 - - GO=1.8.x MONGODB=x86_64-3.4.8 + - MONGODB=x86_64-ubuntu1404-3.0.15 + - MONGODB=x86_64-ubuntu1404-3.2.17 + - MONGODB=x86_64-ubuntu1404-3.4.10 + - MONGODB=x86_64-ubuntu1404-3.6.0 install: - - eval "$(gimme $GO)" - - wget $BUCKET/mongodb-linux-$MONGODB.tgz + - wget $FASTDL/mongodb-linux-$MONGODB.tgz - tar xzvf mongodb-linux-$MONGODB.tgz - export PATH=$PWD/mongodb-linux-$MONGODB/bin:$PATH diff --git a/README.md b/README.md index 87cde972e..c605e6bb0 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ Further PR's (with tests) are welcome, but please maintain backwards compatibili * Fixes attempting to authenticate before every query ([details](https://github.com/go-mgo/mgo/issues/254)) * Removes bulk update / delete batch size limitations ([details](https://github.com/go-mgo/mgo/issues/288)) * Adds native support for `time.Duration` marshalling ([details](https://github.com/go-mgo/mgo/pull/373)) -* Reduce memory footprint / garbage collection pressure by reusing buffers ([details](https://github.com/go-mgo/mgo/pull/229)) +* Reduce memory footprint / garbage collection pressure by reusing buffers ([details](https://github.com/go-mgo/mgo/pull/229), [more](https://github.com/globalsign/mgo/pull/56)) * Support majority read concerns ([details](https://github.com/globalsign/mgo/pull/2)) * Improved connection handling ([details](https://github.com/globalsign/mgo/pull/5)) * Hides SASL warnings ([details](https://github.com/globalsign/mgo/pull/7)) @@ -32,10 +32,13 @@ Further PR's (with tests) are welcome, but please maintain backwards compatibili * GetBSON correctly handles structs with both fields and pointers ([details](https://github.com/globalsign/mgo/pull/40)) * Improved bson.Raw unmarshalling performance ([details](https://github.com/globalsign/mgo/pull/49)) * Minimise socket connection timeouts due to excessive locking ([details](https://github.com/globalsign/mgo/pull/52)) +* Natively support X509 client authentication ([details](https://github.com/globalsign/mgo/pull/55)) +* Gracefully recover from a temporarily unreachable server ([details](https://github.com/globalsign/mgo/pull/69)) --- ### Thanks to +* @bachue * @bozaro * @BenLubar * @carter2000 diff --git a/auth_test.go b/auth_test.go index ed1af5abf..689812477 100644 --- a/auth_test.go +++ b/auth_test.go @@ -28,6 +28,7 @@ package mgo_test import ( "crypto/tls" + "crypto/x509" "flag" "fmt" "io/ioutil" @@ -38,7 +39,7 @@ import ( "sync" "time" - mgo "github.com/globalsign/mgo" + "github.com/globalsign/mgo" . "gopkg.in/check.v1" ) @@ -963,6 +964,73 @@ func (s *S) TestAuthX509Cred(c *C) { c.Assert(len(names) > 0, Equals, true) } +func (s *S) TestAuthX509CredRDNConstruction(c *C) { + session, err := mgo.Dial("localhost:40001") + c.Assert(err, IsNil) + defer session.Close() + binfo, err := session.BuildInfo() + c.Assert(err, IsNil) + if binfo.OpenSSLVersion == "" { + c.Skip("server does not support SSL") + } + + clientCertPEM, err := ioutil.ReadFile("harness/certs/client.pem") + c.Assert(err, IsNil) + + clientCert, err := tls.X509KeyPair(clientCertPEM, clientCertPEM) + c.Assert(err, IsNil) + + clientCert.Leaf, err = x509.ParseCertificate(clientCert.Certificate[0]) + c.Assert(err, IsNil) + + tlsConfig := &tls.Config{ + InsecureSkipVerify: true, + Certificates: []tls.Certificate{clientCert}, + } + + var host = "localhost:40003" + c.Logf("Connecting to %s...", host) + session, err = mgo.DialWithInfo(&mgo.DialInfo{ + Addrs: []string{host}, + DialServer: func(addr *mgo.ServerAddr) (net.Conn, error) { + return tls.Dial("tcp", addr.String(), tlsConfig) + }, + }) + c.Assert(err, IsNil) + defer session.Close() + + cred := &mgo.Credential{ + Username: "root", + Mechanism: "MONGODB-X509", + Source: "$external", + Certificate: tlsConfig.Certificates[0].Leaf, + } + err = session.Login(cred) + c.Assert(err, NotNil) + + err = session.Login(&mgo.Credential{Username: "root", Password: "rapadura"}) + c.Assert(err, IsNil) + + // This needs to be kept in sync with client.pem + x509Subject := "CN=localhost,OU=Client,O=MGO,L=MGO,ST=MGO,C=GO" + + externalDB := session.DB("$external") + var x509User = mgo.User{ + Username: x509Subject, + OtherDBRoles: map[string][]mgo.Role{"admin": {mgo.RoleRoot}}, + } + err = externalDB.UpsertUser(&x509User) + c.Assert(err, IsNil) + + session.LogoutAll() + + cred.Username = "" + c.Logf("Authenticating...") + err = session.Login(cred) + c.Assert(err, IsNil) + c.Logf("Authenticated!") +} + var ( plainFlag = flag.String("plain", "", "Host to test PLAIN authentication against (depends on custom environment)") plainUser = "einstein" diff --git a/bulk.go b/bulk.go index d6925fba4..c234fccee 100644 --- a/bulk.go +++ b/bulk.go @@ -3,6 +3,7 @@ package mgo import ( "bytes" "sort" + "sync" "github.com/globalsign/mgo/bson" ) @@ -118,6 +119,15 @@ func (e *BulkError) Cases() []BulkErrorCase { return e.ecases } +var actionPool = sync.Pool{ + New: func() interface{} { + return &bulkAction{ + docs: make([]interface{}, 0), + idxs: make([]int, 0), + } + }, +} + // Bulk returns a value to prepare the execution of a bulk operation. func (c *Collection) Bulk() *Bulk { return &Bulk{c: c, ordered: true} @@ -145,7 +155,9 @@ func (b *Bulk) action(op bulkOp, opcount int) *bulkAction { } } if action == nil { - b.actions = append(b.actions, bulkAction{op: op}) + a := actionPool.Get().(*bulkAction) + a.op = op + b.actions = append(b.actions, *a) action = &b.actions[len(b.actions)-1] } for i := 0; i < opcount; i++ { @@ -288,6 +300,9 @@ func (b *Bulk) Run() (*BulkResult, error) { default: panic("unknown bulk operation") } + action.idxs = action.idxs[0:0] + action.docs = action.docs[0:0] + actionPool.Put(action) if !ok { failed = true if b.ordered { diff --git a/cluster_test.go b/cluster_test.go index 539422be7..8945e0962 100644 --- a/cluster_test.go +++ b/cluster_test.go @@ -2083,8 +2083,8 @@ func (s *S) TestDoNotFallbackToMonotonic(c *C) { if !s.versionAtLeast(3, 0) { c.Skip("command-counting logic depends on 3.0+") } - if s.versionAtLeast(3, 4) { - c.Skip("failing on 3.4+") + if s.versionAtLeast(3, 2, 17) { + c.Skip("failing on 3.2.17+") } session, err := mgo.Dial("localhost:40012") diff --git a/example_test.go b/example_test.go new file mode 100644 index 000000000..bf7982a46 --- /dev/null +++ b/example_test.go @@ -0,0 +1,136 @@ +package mgo + +import ( + "crypto/tls" + "crypto/x509" + "io/ioutil" + "net" + "sync" +) + +func ExampleCredential_x509Authentication() { + // MongoDB follows RFC2253 for the ordering of the DN - if the order is + // incorrect when creating the user in Mongo, the client will not be able to + // connect. + // + // The best way to generate the DN with the correct ordering is with + // openssl: + // + // openssl x509 -in client.crt -inform PEM -noout -subject -nameopt RFC2253 + // subject= CN=Example App,OU=MongoDB Client Authentication,O=GlobalSign,C=GB + // + // + // And then create the user in MongoDB with the above DN: + // + // db.getSiblingDB("$external").runCommand({ + // createUser: "CN=Example App,OU=MongoDB Client Authentication,O=GlobalSign,C=GB", + // roles: [ + // { role: 'readWrite', db: 'bananas' }, + // { role: 'userAdminAnyDatabase', db: 'admin' } + // ], + // writeConcern: { w: "majority" , wtimeout: 5000 } + // }) + // + // + // References: + // - https://docs.mongodb.com/manual/tutorial/configure-x509-client-authentication/ + // - https://docs.mongodb.com/manual/core/security-x.509/ + // + + // Read in the PEM encoded X509 certificate. + // + // See the client.pem file at the path below. + clientCertPEM, err := ioutil.ReadFile("harness/certs/client.pem") + + // Read in the PEM encoded private key. + clientKeyPEM, err := ioutil.ReadFile("harness/certs/client.key") + + // Parse the private key, and the public key contained within the + // certificate. + clientCert, err := tls.X509KeyPair(clientCertPEM, clientKeyPEM) + + // Parse the actual certificate data + clientCert.Leaf, err = x509.ParseCertificate(clientCert.Certificate[0]) + + // Use the cert to set up a TLS connection to Mongo + tlsConfig := &tls.Config{ + Certificates: []tls.Certificate{clientCert}, + + // This is set to true so the example works within the test + // environment. + // + // DO NOT set InsecureSkipVerify to true in a production + // environment - if you use an untrusted CA/have your own, load + // its certificate into the RootCAs value instead. + // + // RootCAs: myCAChain, + InsecureSkipVerify: true, + } + + // Connect to Mongo using TLS + host := "localhost:40003" + session, err := DialWithInfo(&DialInfo{ + Addrs: []string{host}, + DialServer: func(addr *ServerAddr) (net.Conn, error) { + return tls.Dial("tcp", host, tlsConfig) + }, + }) + + // Authenticate using the certificate + cred := &Credential{Certificate: tlsConfig.Certificates[0].Leaf} + if err := session.Login(cred); err != nil { + panic(err) + } + + // Done! Use mgo as normal from here. + // + // You should actually check the error code at each step. + _ = err +} + +func ExampleSession_concurrency() { + // This example shows the best practise for concurrent use of a mgo session. + // + // Internally mgo maintains a connection pool, dialling new connections as + // required. + // + // Some general suggestions: + // - Define a struct holding the original session, database name and + // collection name instead of passing them explicitly. + // - Define an interface abstracting your data access instead of exposing + // mgo to your application code directly. + // - Limit concurrency at the application level, not with SetPoolLimit(). + + // This will be our concurrent worker + var doStuff = func(wg *sync.WaitGroup, session *Session) { + defer wg.Done() + + // Copy the session - if needed this will dial a new connection which + // can later be reused. + // + // Calling close returns the connection to the pool. + conn := session.Copy() + defer conn.Close() + + // Do something(s) with the connection + _, _ = conn.DB("").C("my_data").Count() + } + + /////////////////////////////////////////////// + + // Dial a connection to Mongo - this creates the connection pool + session, err := Dial("localhost:40003/my_database") + if err != nil { + panic(err) + } + + // Concurrently do things, passing the session to the worker + wg := &sync.WaitGroup{} + for i := 0; i < 10; i++ { + wg.Add(1) + go doStuff(wg, session) + } + wg.Wait() + + session.Close() +} \ No newline at end of file diff --git a/harness/daemons/.env b/harness/daemons/.env index b9a900647..7ba8cf599 100644 --- a/harness/daemons/.env +++ b/harness/daemons/.env @@ -22,10 +22,8 @@ versionAtLeast() { COMMONDOPTSNOIP=" --nohttpinterface - --noprealloc --nojournal --smallfiles - --nssize=1 --oplogSize=1 --dbpath ./db " @@ -55,14 +53,12 @@ if versionAtLeast 3 2; then # 3.2 doesn't like --nojournal on config servers. COMMONCOPTS="$(echo "$COMMONCOPTS" | sed '/--nojournal/d')" - if versionAtLeast 3 4; then # http interface is disabled by default, this option does not exist anymore COMMONDOPTSNOIP="$(echo "$COMMONDOPTSNOIP" | sed '/--nohttpinterface/d')" COMMONDOPTS="$(echo "$COMMONDOPTS" | sed '/--nohttpinterface/d')" COMMONCOPTS="$(echo "$COMMONCOPTS" | sed '/--nohttpinterface/d')" - # config server need to be started as replica set CFG1OPTS="--replSet conf1" CFG2OPTS="--replSet conf2" @@ -71,12 +67,6 @@ if versionAtLeast 3 2; then MONGOS1OPTS="--configdb conf1/127.0.0.1:40101" MONGOS2OPTS="--configdb conf2/127.0.0.1:40102" MONGOS3OPTS="--configdb conf3/127.0.0.1:40103" - else - - # Go back to MMAPv1 so it's not super sluggish. :-( - COMMONDOPTSNOIP="--storageEngine=mmapv1 $COMMONDOPTSNOIP" - COMMONDOPTS="--storageEngine=mmapv1 $COMMONDOPTS" - COMMONCOPTS="--storageEngine=mmapv1 $COMMONCOPTS" fi fi diff --git a/harness/daemons/db2/run b/harness/daemons/db2/run index 5c7b1aa50..e0ec381b3 100755 --- a/harness/daemons/db2/run +++ b/harness/daemons/db2/run @@ -3,6 +3,5 @@ . ../.env exec mongod $COMMONDOPTS \ - --shardsvr \ --port 40002 \ --auth diff --git a/harness/daemons/db3/run b/harness/daemons/db3/run index 539da5fb2..a3788734e 100755 --- a/harness/daemons/db3/run +++ b/harness/daemons/db3/run @@ -3,7 +3,6 @@ . ../.env exec mongod $COMMONDOPTS \ - --shardsvr \ --port 40003 \ --auth \ --sslMode preferSSL \ diff --git a/harness/mongojs/dropall.js b/harness/mongojs/dropall.js index 7fa39d112..ebdf820ef 100644 --- a/harness/mongojs/dropall.js +++ b/harness/mongojs/dropall.js @@ -16,26 +16,33 @@ for (var i in ports) { for (var j in auth) { if (auth[j] == port) { - admin.auth("root", "rapadura") - admin.system.users.find().forEach(function(u) { - if (u.user == "root" || u.user == "reader") { - return; + print("removing user for port " + auth[j]) + for (var k = 0; k < 10; k++) { + var ok = admin.auth("root", "rapadura") + if (ok) { + admin.system.users.find().forEach(function (u) { + if (u.user == "root" || u.user == "reader") { + return; + } + if (typeof admin.dropUser == "function") { + mongo.getDB(u.db).dropUser(u.user); + } else { + admin.removeUser(u.user); + } + }) + break } - if (typeof admin.dropUser == "function") { - mongo.getDB(u.db).dropUser(u.user); - } else { - admin.removeUser(u.user); - } - }) - break + print("failed to auth for port " + port + " retrying in 1s ") + sleep(1000) + } } } - var result = admin.runCommand({"listDatabases": 1}) + var result = admin.runCommand({ "listDatabases": 1 }) for (var j = 0; j != 100; j++) { if (typeof result.databases != "undefined" || notMaster(result)) { break } - result = admin.runCommand({"listDatabases": 1}) + result = admin.runCommand({ "listDatabases": 1 }) } if (notMaster(result)) { continue @@ -49,18 +56,18 @@ for (var i in ports) { for (var j = 0; j != dbs.length; j++) { var db = dbs[j] switch (db.name) { - case "admin": - case "local": - case "config": - break - default: - mongo.getDB(db.name).dropDatabase() + case "admin": + case "local": + case "config": + break + default: + mongo.getDB(db.name).dropDatabase() } } } function notMaster(result) { - return typeof result.errmsg != "undefined" && (result.errmsg.indexOf("not master") >= 0 || result.errmsg.indexOf("no master found")) + return typeof result.errmsg != "undefined" && (result.errmsg.indexOf("not master") >= 0 || result.errmsg.indexOf("no master found")) } // vim:ts=4:sw=4:et diff --git a/harness/mongojs/init.js b/harness/mongojs/init.js index 909cf5162..a5ee1c0e8 100644 --- a/harness/mongojs/init.js +++ b/harness/mongojs/init.js @@ -2,37 +2,42 @@ var settings = {}; // We know the master of the first set (pri=1), but not of the second. -var rs1cfg = {_id: "rs1", - members: [{_id: 1, host: "127.0.0.1:40011", priority: 1, tags: {rs1: "a"}}, - {_id: 2, host: "127.0.0.1:40012", priority: 0, tags: {rs1: "b"}}, - {_id: 3, host: "127.0.0.1:40013", priority: 0, tags: {rs1: "c"}}], - settings: settings} -var rs2cfg = {_id: "rs2", - members: [{_id: 1, host: "127.0.0.1:40021", priority: 1, tags: {rs2: "a"}}, - {_id: 2, host: "127.0.0.1:40022", priority: 1, tags: {rs2: "b"}}, - {_id: 3, host: "127.0.0.1:40023", priority: 1, tags: {rs2: "c"}}], - settings: settings} -var rs3cfg = {_id: "rs3", - members: [{_id: 1, host: "127.0.0.1:40031", priority: 1, tags: {rs3: "a"}}, - {_id: 2, host: "127.0.0.1:40032", priority: 1, tags: {rs3: "b"}}, - {_id: 3, host: "127.0.0.1:40033", priority: 1, tags: {rs3: "c"}}], - settings: settings} +var rs1cfg = { + _id: "rs1", + members: [{ _id: 1, host: "127.0.0.1:40011", priority: 1, tags: { rs1: "a" } }, + { _id: 2, host: "127.0.0.1:40012", priority: 0, tags: { rs1: "b" } }, + { _id: 3, host: "127.0.0.1:40013", priority: 0, tags: { rs1: "c" } }], + settings: settings +} +var rs2cfg = { + _id: "rs2", + members: [{ _id: 1, host: "127.0.0.1:40021", priority: 1, tags: { rs2: "a" } }, + { _id: 2, host: "127.0.0.1:40022", priority: 1, tags: { rs2: "b" } }, + { _id: 3, host: "127.0.0.1:40023", priority: 1, tags: { rs2: "c" } }], + settings: settings +} +var rs3cfg = { + _id: "rs3", + members: [{ _id: 1, host: "127.0.0.1:40031", priority: 1, tags: { rs3: "a" } }, + { _id: 2, host: "127.0.0.1:40032", priority: 0, tags: { rs3: "b" } }, + { _id: 3, host: "127.0.0.1:40033", priority: 0, tags: { rs3: "c" } }], + settings: settings +} for (var i = 0; i != 60; i++) { - try { - db1 = new Mongo("127.0.0.1:40001").getDB("admin") - db2 = new Mongo("127.0.0.1:40002").getDB("admin") - rs1a = new Mongo("127.0.0.1:40011").getDB("admin") - rs2a = new Mongo("127.0.0.1:40021").getDB("admin") - rs3a = new Mongo("127.0.0.1:40031").getDB("admin") + try { + db1 = new Mongo("127.0.0.1:40001").getDB("admin") + rs1a = new Mongo("127.0.0.1:40011").getDB("admin") + rs2a = new Mongo("127.0.0.1:40021").getDB("admin") + rs3a = new Mongo("127.0.0.1:40031").getDB("admin") cfg1 = new Mongo("127.0.0.1:40101").getDB("admin") cfg2 = new Mongo("127.0.0.1:40102").getDB("admin") cfg3 = new Mongo("127.0.0.1:40103").getDB("admin") - break - } catch(err) { - print("Can't connect yet...") - } - sleep(1000) + break + } catch (err) { + print("Can't connect yet...") + } + sleep(1000) } function hasSSL() { @@ -52,28 +57,19 @@ function versionAtLeast() { return true } -rs1a.runCommand({replSetInitiate: rs1cfg}) -rs2a.runCommand({replSetInitiate: rs2cfg}) -rs3a.runCommand({replSetInitiate: rs3cfg}) -if (versionAtLeast(3,4)) { - print("configuring config server for mongodb 3.4") - cfg1.runCommand({replSetInitiate: {_id:"conf1", members: [{"_id":1, "host":"localhost:40101"}]}}) - cfg2.runCommand({replSetInitiate: {_id:"conf2", members: [{"_id":1, "host":"localhost:40102"}]}}) - cfg3.runCommand({replSetInitiate: {_id:"conf3", members: [{"_id":1, "host":"localhost:40103"}]}}) +if (versionAtLeast(3, 4)) { + print("configuring config server for mongodb 3.4+") + cfg1.runCommand({ replSetInitiate: { _id: "conf1", configsvr: true, members: [{ "_id": 1, "host": "localhost:40101" }] } }) + cfg2.runCommand({ replSetInitiate: { _id: "conf2", configsvr: true, members: [{ "_id": 1, "host": "localhost:40102" }] } }) + cfg3.runCommand({ replSetInitiate: { _id: "conf3", configsvr: true, members: [{ "_id": 1, "host": "localhost:40103" }] } }) } -function configShards() { - s1 = new Mongo("127.0.0.1:40201").getDB("admin") - s1.runCommand({addshard: "127.0.0.1:40001"}) - s1.runCommand({addshard: "rs1/127.0.0.1:40011"}) - - s2 = new Mongo("127.0.0.1:40202").getDB("admin") - s2.runCommand({addshard: "rs2/127.0.0.1:40021"}) +sleep(3000) - s3 = new Mongo("127.0.0.1:40203").getDB("admin") - s3.runCommand({addshard: "rs3/127.0.0.1:40031"}) -} +rs1a.runCommand({ replSetInitiate: rs1cfg }) +rs2a.runCommand({ replSetInitiate: rs2cfg }) +rs3a.runCommand({ replSetInitiate: rs3cfg }) function configAuth() { var addrs = ["127.0.0.1:40002", "127.0.0.1:40203", "127.0.0.1:40031"] @@ -83,21 +79,26 @@ function configAuth() { for (var i in addrs) { print("Configuring auth for", addrs[i]) var db = new Mongo(addrs[i]).getDB("admin") - var v = db.serverBuildInfo().versionArray var timedOut = false - if (v < [2, 5]) { - db.addUser("root", "rapadura") - } else { + createUser: + for (var i = 0; i < 60; i++) { try { - db.createUser({user: "root", pwd: "rapadura", roles: ["root"]}) + db.createUser({ user: "root", pwd: "rapadura", roles: ["root"] }) } catch (err) { // 3.2 consistently fails replication of creds on 40031 (config server) print("createUser command returned an error: " + err) if (String(err).indexOf("timed out") >= 0) { timedOut = true; } + // on 3.6 cluster with keyFile, we sometimes get this error + if (String(err).indexOf("Cache Reader No keys found for HMAC that is valid for time")) { + sleep(500) + continue createUser; + } } + break; } + for (var i = 0; i < 60; i++) { var ok = db.auth("root", "rapadura") if (ok || !timedOut) { @@ -105,18 +106,47 @@ function configAuth() { } sleep(1000); } - if (v >= [2, 6]) { - db.createUser({user: "reader", pwd: "rapadura", roles: ["readAnyDatabase"]}) - } else if (v >= [2, 4]) { - db.addUser({user: "reader", pwd: "rapadura", roles: ["readAnyDatabase"]}) - } else { - db.addUser("reader", "rapadura", true) + sleep(500) + db.createUser({ user: "reader", pwd: "rapadura", roles: ["readAnyDatabase"] }) + sleep(3000) + } +} + +function addShard(adminDb, shardList) { + for (var index = 0; index < shardList.length; index++) { + for (var i = 0; i < 10; i++) { + var result = adminDb.runCommand({ addshard: shardList[index] }) + if (result.ok == 1) { + print("shard " + shardList[index] + " sucessfully added") + break + } else { + print("fail to add shard: " + shardList[index] + " error: " + JSON.stringify(result) + ", retrying in 1s") + sleep(1000) + } } } } +function configShards() { + s1 = new Mongo("127.0.0.1:40201").getDB("admin") + addShard(s1, ["127.0.0.1:40001", "rs1/127.0.0.1:40011"]) + + s2 = new Mongo("127.0.0.1:40202").getDB("admin") + addShard(s2, ["rs2/127.0.0.1:40021"]) + + s3 = new Mongo("127.0.0.1:40203").getDB("admin") + for (var i = 0; i < 10; i++) { + var ok = s3.auth("root", "rapadura") + if (ok) { + break + } + sleep(1000) + } + addShard(s3, ["rs3/127.0.0.1:40031"]) +} + function countHealthy(rs) { - var status = rs.runCommand({replSetGetStatus: 1}) + var status = rs.runCommand({ replSetGetStatus: 1 }) var count = 0 var primary = 0 if (typeof status.members != "undefined") { @@ -131,7 +161,7 @@ function countHealthy(rs) { } } if (primary == 0) { - count = 0 + count = 0 } return count } @@ -142,8 +172,9 @@ for (var i = 0; i != 60; i++) { var count = countHealthy(rs1a) + countHealthy(rs2a) + countHealthy(rs3a) print("Replica sets have", count, "healthy nodes.") if (count == totalRSMembers) { - configShards() configAuth() + sleep(2000) + configShards() quit(0) } sleep(1000) diff --git a/session.go b/session.go index 074f48688..b62707c84 100644 --- a/session.go +++ b/session.go @@ -28,6 +28,9 @@ package mgo import ( "crypto/md5" + "crypto/x509" + "crypto/x509/pkix" + "encoding/asn1" "encoding/hex" "errors" "fmt" @@ -825,6 +828,14 @@ type Credential struct { // Mechanism defines the protocol for credential negotiation. // Defaults to "MONGODB-CR". Mechanism string + + // Certificate sets the x509 certificate for authentication, see: + // + // https://docs.mongodb.com/manual/tutorial/configure-x509-client-authentication/ + // + // If using certificate authentication the Username, Mechanism and Source + // fields should not be set. + Certificate *x509.Certificate } // Login authenticates with MongoDB using the provided credential. The @@ -847,6 +858,19 @@ func (s *Session) Login(cred *Credential) error { defer socket.Release() credCopy := *cred + if cred.Certificate != nil && cred.Username != "" { + return errors.New("failed to login, both certificate and credentials are given") + } + + if cred.Certificate != nil { + credCopy.Username, err = getRFC2253NameStringFromCert(cred.Certificate) + if err != nil { + return err + } + credCopy.Mechanism = "MONGODB-X509" + credCopy.Source = "$external" + } + if cred.Source == "" { if cred.Mechanism == "GSSAPI" { credCopy.Source = "$external" @@ -4745,13 +4769,13 @@ func (s *Session) acquireSocket(slaveOk bool) (*mongoSocket, error) { s.m.RLock() // If there is a slave socket reserved and its use is acceptable, take it as long // as there isn't a master socket which would be preferred by the read preference mode. - if s.slaveSocket != nil && s.slaveOk && slaveOk && (s.masterSocket == nil || s.consistency != PrimaryPreferred && s.consistency != Monotonic) { + if s.slaveSocket != nil && s.slaveSocket.dead == nil && s.slaveOk && slaveOk && (s.masterSocket == nil || s.consistency != PrimaryPreferred && s.consistency != Monotonic) { socket := s.slaveSocket socket.Acquire() s.m.RUnlock() return socket, nil } - if s.masterSocket != nil { + if s.masterSocket != nil && s.masterSocket.dead == nil { socket := s.masterSocket socket.Acquire() s.m.RUnlock() @@ -4765,12 +4789,20 @@ func (s *Session) acquireSocket(slaveOk bool) (*mongoSocket, error) { defer s.m.Unlock() if s.slaveSocket != nil && s.slaveOk && slaveOk && (s.masterSocket == nil || s.consistency != PrimaryPreferred && s.consistency != Monotonic) { - s.slaveSocket.Acquire() - return s.slaveSocket, nil + if s.slaveSocket.dead == nil { + s.slaveSocket.Acquire() + return s.slaveSocket, nil + } else { + s.unsetSocket() + } } if s.masterSocket != nil { - s.masterSocket.Acquire() - return s.masterSocket, nil + if s.masterSocket.dead == nil { + s.masterSocket.Acquire() + return s.masterSocket, nil + } else { + s.unsetSocket() + } } // Still not good. We need a new socket. @@ -4821,9 +4853,11 @@ func (s *Session) setSocket(socket *mongoSocket) { // unsetSocket releases any slave and/or master sockets reserved. func (s *Session) unsetSocket() { if s.masterSocket != nil { + debugf("unset master socket from session %p", s) s.masterSocket.Release() } if s.slaveSocket != nil { + debugf("unset slave socket from session %p", s) s.slaveSocket.Release() } s.masterSocket = nil @@ -5212,3 +5246,73 @@ func hasErrMsg(d []byte) bool { } return false } + +// getRFC2253NameStringFromCert converts from an ASN.1 structured representation of the certificate +// to a UTF-8 string representation(RDN) and returns it. +func getRFC2253NameStringFromCert(certificate *x509.Certificate) (string, error) { + var RDNElements = pkix.RDNSequence{} + _, err := asn1.Unmarshal(certificate.RawSubject, &RDNElements) + return getRFC2253NameString(&RDNElements), err +} + +// getRFC2253NameString converts from an ASN.1 structured representation of the RDNSequence +// from the certificate to a UTF-8 string representation(RDN) and returns it. +func getRFC2253NameString(RDNElements *pkix.RDNSequence) string { + var RDNElementsString = []string{} + var replacer = strings.NewReplacer(",", "\\,", "=", "\\=", "+", "\\+", "<", "\\<", ">", "\\>", ";", "\\;") + //The elements in the sequence needs to be reversed when converting them + for i := len(*RDNElements) - 1; i >= 0; i-- { + var nameAndValueList = make([]string,len((*RDNElements)[i])) + for j, attribute := range (*RDNElements)[i] { + var shortAttributeName = rdnOIDToShortName(attribute.Type) + if len(shortAttributeName) <= 0 { + nameAndValueList[j] = fmt.Sprintf("%s=%X", attribute.Type.String(), attribute.Value.([]byte)) + continue + } + var attributeValueString = attribute.Value.(string) + // escape leading space or # + if strings.HasPrefix(attributeValueString, " ") || strings.HasPrefix(attributeValueString, "#") { + attributeValueString = "\\" + attributeValueString + } + // escape trailing space, unless it's already escaped + if strings.HasSuffix(attributeValueString, " ") && !strings.HasSuffix(attributeValueString, "\\ ") { + attributeValueString = attributeValueString[:len(attributeValueString)-1] + "\\ " + } + + // escape , = + < > # ; + attributeValueString = replacer.Replace(attributeValueString) + nameAndValueList[j] = fmt.Sprintf("%s=%s", shortAttributeName, attributeValueString) + } + + RDNElementsString = append(RDNElementsString, strings.Join(nameAndValueList, "+")) + } + + return strings.Join(RDNElementsString, ",") +} + +var oidsToShortNames = []struct { + oid asn1.ObjectIdentifier + shortName string +}{ + {asn1.ObjectIdentifier{2, 5, 4, 3}, "CN"}, + {asn1.ObjectIdentifier{2, 5, 4, 6}, "C"}, + {asn1.ObjectIdentifier{2, 5, 4, 7}, "L"}, + {asn1.ObjectIdentifier{2, 5, 4, 8}, "ST"}, + {asn1.ObjectIdentifier{2, 5, 4, 10}, "O"}, + {asn1.ObjectIdentifier{2, 5, 4, 11}, "OU"}, + {asn1.ObjectIdentifier{2, 5, 4, 9}, "STREET"}, + {asn1.ObjectIdentifier{0, 9, 2342, 19200300, 100, 1, 25}, "DC"}, + {asn1.ObjectIdentifier{0, 9, 2342, 19200300, 100, 1, 1}, "UID"}, +} + +// rdnOIDToShortName returns an short name of the given RDN OID. If the OID does not have a short +// name, the function returns an empty string +func rdnOIDToShortName(oid asn1.ObjectIdentifier) string { + for i := range oidsToShortNames { + if oidsToShortNames[i].oid.Equal(oid) { + return oidsToShortNames[i].shortName + } + } + + return "" +} diff --git a/session_internal_test.go b/session_internal_test.go index f5f796c99..ddce59cae 100644 --- a/session_internal_test.go +++ b/session_internal_test.go @@ -1,11 +1,17 @@ package mgo import ( - "testing" - + "crypto/x509/pkix" + "encoding/asn1" "github.com/globalsign/mgo/bson" + . "gopkg.in/check.v1" + "testing" ) +type S struct{} + +var _ = Suite(&S{}) + // This file is for testing functions that are not exported outside the mgo // package - avoid doing so if at all possible. @@ -22,3 +28,37 @@ func TestIndexedInt64FieldsBug(t *testing.T) { _ = simpleIndexKey(input) } + +func (s *S) TestGetRFC2253NameStringSingleValued(c *C) { + var RDNElements = pkix.RDNSequence{ + {{Type: asn1.ObjectIdentifier{2, 5, 4, 6}, Value: "GO"}}, + {{Type: asn1.ObjectIdentifier{2, 5, 4, 8}, Value: "MGO"}}, + {{Type: asn1.ObjectIdentifier{2, 5, 4, 7}, Value: "MGO"}}, + {{Type: asn1.ObjectIdentifier{2, 5, 4, 10}, Value: "MGO"}}, + {{Type: asn1.ObjectIdentifier{2, 5, 4, 11}, Value: "Client"}}, + {{Type: asn1.ObjectIdentifier{2, 5, 4, 3}, Value: "localhost"}}, + } + + c.Assert(getRFC2253NameString(&RDNElements), Equals, "CN=localhost,OU=Client,O=MGO,L=MGO,ST=MGO,C=GO") +} + +func (s *S) TestGetRFC2253NameStringEscapeChars(c *C) { + var RDNElements = pkix.RDNSequence{ + {{Type: asn1.ObjectIdentifier{2, 5, 4, 6}, Value: "GB"}}, + {{Type: asn1.ObjectIdentifier{2, 5, 4, 8}, Value: "MGO "}}, + {{Type: asn1.ObjectIdentifier{2, 5, 4, 10}, Value: "Sue, Grabbit and Runn < > ;"}}, + {{Type: asn1.ObjectIdentifier{2, 5, 4, 3}, Value: "L. Eagle"}}, + } + + c.Assert(getRFC2253NameString(&RDNElements), Equals, "CN=L. Eagle,O=Sue\\, Grabbit and Runn \\< \\> \\;,ST=MGO\\ ,C=GB") +} + +func (s *S) TestGetRFC2253NameStringMultiValued(c *C) { + var RDNElements = pkix.RDNSequence{ + {{Type: asn1.ObjectIdentifier{2, 5, 4, 6}, Value: "US"}}, + {{Type: asn1.ObjectIdentifier{2, 5, 4, 10}, Value: "Widget Inc."}}, + {{Type: asn1.ObjectIdentifier{2, 5, 4, 11}, Value: "Sales"}, {Type: asn1.ObjectIdentifier{2, 5, 4, 3}, Value: "J. Smith"}}, + } + + c.Assert(getRFC2253NameString(&RDNElements), Equals, "OU=Sales+CN=J. Smith,O=Widget Inc.,C=US") +} diff --git a/session_test.go b/session_test.go index 227052719..eb2c812b3 100644 --- a/session_test.go +++ b/session_test.go @@ -413,19 +413,11 @@ func (s *S) TestDatabaseAndCollectionNames(c *C) { names, err = db1.CollectionNames() c.Assert(err, IsNil) - if s.versionAtLeast(3, 4) { - c.Assert(names, DeepEquals, []string{"col1", "col2"}) - } else { - c.Assert(names, DeepEquals, []string{"col1", "col2", "system.indexes"}) - } + c.Assert(filterDBs(names), DeepEquals, []string{"col1", "col2"}) names, err = db2.CollectionNames() c.Assert(err, IsNil) - if s.versionAtLeast(3, 4) { - c.Assert(names, DeepEquals, []string{"col3"}) - } else { - c.Assert(names, DeepEquals, []string{"col3", "system.indexes"}) - } + c.Assert(filterDBs(names), DeepEquals, []string{"col3"}) } func (s *S) TestSelect(c *C) { @@ -854,7 +846,7 @@ func filterDBs(dbs []string) []string { var i int for _, name := range dbs { switch name { - case "admin", "local": + case "admin", "local", "config", "system.indexes": default: dbs[i] = name i++ @@ -880,22 +872,14 @@ func (s *S) TestDropCollection(c *C) { names, err := db.CollectionNames() c.Assert(err, IsNil) - if s.versionAtLeast(3, 4) { - c.Assert(names, DeepEquals, []string{"col2"}) - } else { - c.Assert(names, DeepEquals, []string{"col2", "system.indexes"}) - } + c.Assert(filterDBs(names), DeepEquals, []string{"col2"}) err = db.C("col2").DropCollection() c.Assert(err, IsNil) names, err = db.CollectionNames() c.Assert(err, IsNil) - if s.versionAtLeast(3, 4) { - c.Assert(len(names), Equals, 0) - } else { - c.Assert(names, DeepEquals, []string{"system.indexes"}) - } + c.Assert(len(filterDBs(names)), Equals, 0) } func (s *S) TestCreateCollectionCapped(c *C) { @@ -1263,19 +1247,13 @@ func (s *S) TestFindAndModifyBug997828(c *C) { result := make(M) _, err = coll.Find(M{"n": "not-a-number"}).Apply(mgo.Change{Update: M{"$inc": M{"n": 1}}}, result) c.Assert(err, ErrorMatches, `(exception: )?Cannot apply \$inc .*`) - if s.versionAtLeast(2, 1) { - qerr, _ := err.(*mgo.QueryError) - c.Assert(qerr, NotNil, Commentf("err: %#v", err)) - if s.versionAtLeast(2, 6) { - // Oh, the dance of error codes. :-( - c.Assert(qerr.Code, Equals, 16837) - } else { - c.Assert(qerr.Code, Equals, 10140) - } + qerr, _ := err.(*mgo.QueryError) + c.Assert(qerr, NotNil, Commentf("err: %#v", err)) + if s.versionAtLeast(3, 6) { + // Oh, the dance of error codes. :-( + c.Assert(qerr.Code, Equals, 14) } else { - lerr, _ := err.(*mgo.LastError) - c.Assert(lerr, NotNil, Commentf("err: %#v", err)) - c.Assert(lerr.Code, Equals, 10140) + c.Assert(qerr.Code, Equals, 16837) } } @@ -1650,7 +1628,7 @@ func (s *S) TestQueryComment(c *C) { db := session.DB("mydb") coll := db.C("mycoll") - err = db.Run(bson.M{"profile": 2}, nil) + err = db.Run(bson.D{{Name: "profile", Value: 2}}, nil) c.Assert(err, IsNil) ns := []int{40, 41, 42} @@ -1672,12 +1650,20 @@ func (s *S) TestQueryComment(c *C) { commentField := "query.$comment" nField := "query.$query.n" if s.versionAtLeast(3, 2) { - commentField = "query.comment" - nField = "query.filter.n" + if s.versionAtLeast(3, 6) { + commentField = "command.comment" + nField = "command.filter.n" + } else { + commentField = "query.comment" + nField = "query.filter.n" + } } n, err := session.DB("mydb").C("system.profile").Find(bson.M{nField: 41, commentField: "some comment"}).Count() c.Assert(err, IsNil) c.Assert(n, Equals, 1) + + err = db.Run(bson.D{{Name: "profile", Value: 0}}, nil) + c.Assert(err, IsNil) } func (s *S) TestFindOneNotFound(c *C) { @@ -2845,9 +2831,6 @@ func (s *S) TestFindForResetsResult(c *C) { } func (s *S) TestFindIterSnapshot(c *C) { - if s.versionAtLeast(3, 2) { - c.Skip("Broken in 3.2: https://jira.mongodb.org/browse/SERVER-21403") - } session, err := mgo.Dial("localhost:40001") c.Assert(err, IsNil) @@ -3528,7 +3511,7 @@ func (s *S) TestEnsureIndex(c *C) { // db.runCommand({"listIndexes": }) // // and iterate over the returned cursor. - if s.versionAtLeast(3, 4) { + if s.versionAtLeast(3, 2, 17) { c.Assert(getIndex34(session, "mydb", "mycoll", test.expected["name"].(string)), DeepEquals, test.expected) } else { idxs := session.DB("mydb").C("system.indexes") @@ -3638,7 +3621,7 @@ func (s *S) TestEnsureIndexKey(c *C) { err = coll.EnsureIndexKey("a") c.Assert(err, IsNil) - if s.versionAtLeast(3, 4) { + if s.versionAtLeast(3, 2, 17) { expected := M{ "name": "a_1", "key": M{"a": 1}, @@ -3703,7 +3686,7 @@ func (s *S) TestEnsureIndexDropIndex(c *C) { err = coll.DropIndex("-b") c.Assert(err, IsNil) - if s.versionAtLeast(3, 4) { + if s.versionAtLeast(3, 2, 17) { // system.indexes is deprecated since 3.0, use // db.runCommand({"listIndexes": }) // instead @@ -3758,7 +3741,7 @@ func (s *S) TestEnsureIndexDropIndexName(c *C) { err = coll.DropIndexName("a") c.Assert(err, IsNil) - if s.versionAtLeast(3, 4) { + if s.versionAtLeast(3, 2, 17) { // system.indexes is deprecated since 3.0, use // db.runCommand({"listIndexes": }) // instead @@ -3814,7 +3797,7 @@ func (s *S) TestEnsureIndexDropAllIndexes(c *C) { err = coll.DropAllIndexes() c.Assert(err, IsNil) - if s.versionAtLeast(3, 4) { + if s.versionAtLeast(3, 2, 17) { // system.indexes is deprecated since 3.0, use // db.runCommand({"listIndexes": }) // instead @@ -4395,8 +4378,8 @@ func (s *S) TestRepairCursor(c *C) { if !s.versionAtLeast(2, 7) { c.Skip("RepairCursor only works on 2.7+") } - if s.versionAtLeast(3, 4) { - c.Skip("fail on 3.4+") + if s.versionAtLeast(3, 2, 17) { + c.Skip("fail on 3.2.17+") } session, err := mgo.Dial("localhost:40001")