Move from Cassandra to Postgres

This commit is contained in:
Jonathan Barrow
2022-09-11 02:22:37 -04:00
parent a3f9757608
commit ca54e576d5
23 changed files with 232 additions and 252 deletions

View File

@@ -2,7 +2,6 @@ package database
import (
"encoding/base64"
"math/rand"
"time"
"github.com/PretendoNetwork/friends-secure/globals"
@@ -16,31 +15,26 @@ func AcceptFriendshipAndReturnFriendInfo(friendRequestID uint64) *nexproto.Frien
var senderPID uint32
var recipientPID uint32
if err := cassandraClusterSession.Query(`SELECT sender_pid, recipient_pid FROM pretendo_friends.friend_requests WHERE id=?`, friendRequestID).Scan(&senderPID, &recipientPID); err != nil {
err := postgres.QueryRow(`SELECT sender_pid, recipient_pid FROM wiiu.friend_requests WHERE id=$1`, friendRequestID).Scan(&senderPID, &recipientPID)
if err != nil {
globals.Logger.Critical(err.Error())
return nil
}
rand.Seed(time.Now().UnixNano())
nodeID := rand.Intn(len(globals.SnowflakeNodes))
snowflakeNode := globals.SnowflakeNodes[nodeID]
friendshipID1 := uint64(snowflakeNode.Generate().Int64())
friendshipID2 := uint64(snowflakeNode.Generate().Int64())
acceptedTime := nex.NewDateTime(0)
acceptedTime.FromTimestamp(time.Now())
// Friendships are two-way relationships, not just one link between 2 entities
// "A" has friend "B" and "B" has friend "A", so store both relationships
if err := cassandraClusterSession.Query(`INSERT INTO pretendo_friends.friendships (id, user1_pid, user2_pid, date) VALUES (?, ?, ?, ?) IF NOT EXISTS`, friendshipID1, senderPID, recipientPID, acceptedTime.Value()).Exec(); err != nil {
_, err = postgres.Exec(`INSERT INTO wiiu.friendships (user1_pid, user2_pid, date) VALUES ($1, $2, $3)`, senderPID, recipientPID, acceptedTime.Value())
if err != nil {
globals.Logger.Critical(err.Error())
return nil
}
if err := cassandraClusterSession.Query(`INSERT INTO pretendo_friends.friendships (id, user1_pid, user2_pid, date) VALUES (?, ?, ?, ?) IF NOT EXISTS`, friendshipID2, recipientPID, senderPID, acceptedTime.Value()).Exec(); err != nil {
_, err = postgres.Exec(`INSERT INTO wiiu.friendships (user1_pid, user2_pid, date) VALUES ($1, $2, $3)`, recipientPID, senderPID, acceptedTime.Value())
if err != nil {
globals.Logger.Critical(err.Error())
return nil
}
@@ -98,12 +92,14 @@ func AcceptFriendshipAndReturnFriendInfo(friendRequestID uint64) *nexproto.Frien
friendInfo.Presence.Unknown7 = 0
var lastOnlineTime uint64
if err := cassandraClusterSession.Query(`SELECT time FROM pretendo_friends.last_online WHERE pid=?`, senderPID).Scan(&lastOnlineTime); err != nil {
err := postgres.QueryRow(`SELECT last_online FROM wiiu.user_data WHERE pid=$1`, senderPID).Scan(&lastOnlineTime)
if err != nil {
lastOnlineTime = nex.NewDateTime(0).Now()
if err == gocql.ErrNotFound {
lastOnlineTime = nex.NewDateTime(0).Now()
globals.Logger.Error(err.Error())
} else {
globals.Logger.Critical(err.Error())
lastOnlineTime = nex.NewDateTime(0).Now()
}
}

View File

@@ -1,124 +0,0 @@
package database
import (
"flag"
"fmt"
"time"
"github.com/PretendoNetwork/friends-secure/globals"
"github.com/gocql/gocql"
)
var cluster *gocql.ClusterConfig
var cassandraClusterSession *gocql.Session
func connectCassandra() {
// Connect to Cassandra
var err error
cluster = gocql.NewCluster("127.0.0.1")
cluster.Timeout = 30 * time.Second
createKeyspace("pretendo_friends")
cluster.Keyspace = "pretendo_friends"
cassandraClusterSession, err = cluster.CreateSession()
if err != nil {
globals.Logger.Critical(err.Error())
return
}
// Create tables if missing
if err := cassandraClusterSession.Query(`CREATE TABLE IF NOT EXISTS pretendo_friends.preferences (
pid int PRIMARY KEY,
show_online boolean,
show_current_game boolean,
block_friend_requests boolean
)`).Exec(); err != nil {
globals.Logger.Critical(err.Error())
return
}
if err := cassandraClusterSession.Query(`CREATE TABLE IF NOT EXISTS pretendo_friends.blocks (
id text PRIMARY KEY,
blocker_pid int,
blocked_pid int,
date bigint
)`).Exec(); err != nil {
globals.Logger.Critical(err.Error())
return
}
if err := cassandraClusterSession.Query(`CREATE TABLE IF NOT EXISTS pretendo_friends.friend_requests (
id bigint PRIMARY KEY,
sender_pid int,
recipient_pid int,
sent_on bigint,
expires_on bigint,
message text,
received boolean,
accepted boolean,
denied boolean
)`).Exec(); err != nil {
globals.Logger.Critical(err.Error())
return
}
if err := cassandraClusterSession.Query(`CREATE TABLE IF NOT EXISTS pretendo_friends.friendships (
id bigint PRIMARY KEY,
user1_pid int,
user2_pid int,
date bigint
)`).Exec(); err != nil {
globals.Logger.Critical(err.Error())
return
}
if err := cassandraClusterSession.Query(`CREATE TABLE IF NOT EXISTS pretendo_friends.comments (
pid int PRIMARY KEY,
message text,
changed bigint
)`).Exec(); err != nil {
globals.Logger.Critical(err.Error())
return
}
if err := cassandraClusterSession.Query(`CREATE TABLE IF NOT EXISTS pretendo_friends.last_online (
pid int PRIMARY KEY,
time bigint
)`).Exec(); err != nil {
globals.Logger.Critical(err.Error())
return
}
globals.Logger.Success("Connected to db")
}
// Adapted from gocql common_test.go
func createKeyspace(keyspace string) {
flagRF := flag.Int("rf", 1, "replication factor for pretendo_friends keyspace")
c := *cluster
c.Keyspace = "system"
c.Timeout = 30 * time.Second
s, err := c.CreateSession()
if err != nil {
globals.Logger.Critical(err.Error())
}
defer s.Close()
if err := s.Query(fmt.Sprintf(`CREATE KEYSPACE IF NOT EXISTS %s
WITH replication = {
'class' : 'SimpleStrategy',
'replication_factor' : %d
}`, keyspace, *flagRF)).Exec(); err != nil {
globals.Logger.Critical(err.Error())
}
}

View File

@@ -0,0 +1,24 @@
package database
import (
"database/sql"
"os"
_ "github.com/lib/pq"
"github.com/PretendoNetwork/friends-secure/globals"
)
var postgres *sql.DB
func connectPostgres() {
var err error
postgres, err = sql.Open("postgres", os.Getenv("DATABASE_URI"))
if err != nil {
globals.Logger.Critical(err.Error())
}
initPostgresWiiU()
// TODO: 3DS database
}

View File

@@ -2,5 +2,5 @@ package database
func Connect() {
connectMongo()
connectCassandra()
connectPostgres()
}

View File

@@ -1,10 +1,11 @@
package database
import (
"database/sql"
"github.com/PretendoNetwork/friends-secure/globals"
"github.com/PretendoNetwork/nex-go"
nexproto "github.com/PretendoNetwork/nex-protocols-go"
"github.com/gocql/gocql"
)
// Get a users comment
@@ -14,12 +15,11 @@ func GetUserComment(pid uint32) *nexproto.Comment {
var changed uint64 = 0
if err := cassandraClusterSession.Query(`SELECT message, changed FROM pretendo_friends.comments WHERE pid=?`,
pid).Consistency(gocql.One).Scan(&comment.Contents, &changed); err != nil {
if err == gocql.ErrNotFound {
comment.Contents = ""
err := postgres.QueryRow(`SELECT comment, comment_changed FROM wiiu.user_data WHERE pid=$1`, pid).Scan(&comment.Contents, &changed)
if err != nil {
if err == sql.ErrNoRows {
globals.Logger.Warning(err.Error())
} else {
comment.Contents = ""
globals.Logger.Critical(err.Error())
}
}

View File

@@ -14,19 +14,18 @@ import (
// Get a users friend list
func GetUserFriendList(pid uint32) []*nexproto.FriendInfo {
var sliceMap []map[string]interface{}
var err error
if sliceMap, err = cassandraClusterSession.Query(`SELECT user2_pid, date FROM pretendo_friends.friendships WHERE user1_pid=? ALLOW FILTERING`, pid).Iter().SliceMap(); err != nil {
globals.Logger.Critical(err.Error())
return make([]*nexproto.FriendInfo, 0)
}
friendList := make([]*nexproto.FriendInfo, 0)
for i := 0; i < len(sliceMap); i++ {
friendPID := uint32(sliceMap[i]["user2_pid"].(int))
rows, err := postgres.Query(`SELECT user2_pid, date FROM wiiu.friendships WHERE user1_pid=$1`, pid)
if err != nil {
globals.Logger.Critical(err.Error())
return friendList
}
for rows.Next() {
var friendPID uint32
var date uint64
rows.Scan(&friendPID, &date)
friendInfo := nexproto.NewFriendInfo()
connectedUser := globals.ConnectedUsers[friendPID]
@@ -91,12 +90,14 @@ func GetUserFriendList(pid uint32) []*nexproto.FriendInfo {
friendInfo.Presence.Unknown7 = 0
var lastOnlineTime uint64
if err := cassandraClusterSession.Query(`SELECT time FROM pretendo_friends.last_online WHERE pid=?`, friendPID).Scan(&lastOnlineTime); err != nil {
err := postgres.QueryRow(`SELECT last_online FROM wiiu.user_data WHERE pid=$1`, friendPID).Scan(&lastOnlineTime)
if err != nil {
lastOnlineTime = nex.NewDateTime(0).Now()
if err == gocql.ErrNotFound {
lastOnlineTime = nex.NewDateTime(0).Now()
globals.Logger.Error(err.Error())
} else {
globals.Logger.Critical(err.Error())
lastOnlineTime = nex.NewDateTime(0).Now()
}
}
@@ -104,7 +105,7 @@ func GetUserFriendList(pid uint32) []*nexproto.FriendInfo {
}
friendInfo.Status = GetUserComment(friendPID)
friendInfo.BecameFriend = nex.NewDateTime(uint64(sliceMap[i]["date"].(int64)))
friendInfo.BecameFriend = nex.NewDateTime(date)
friendInfo.LastOnline = lastOnline
friendInfo.Unknown = 0

View File

@@ -11,18 +11,22 @@ import (
// Get a users received friend requests
func GetUserFriendRequestsIn(pid uint32) []*nexproto.FriendRequest {
var sliceMap []map[string]interface{}
var err error
if sliceMap, err = cassandraClusterSession.Query(`SELECT id, sender_pid, sent_on, expires_on, message, received FROM pretendo_friends.friend_requests WHERE recipient_pid=? AND accepted=false AND denied=false ALLOW FILTERING`, pid).Iter().SliceMap(); err != nil {
globals.Logger.Critical(err.Error())
return make([]*nexproto.FriendRequest, 0)
}
friendRequestsIn := make([]*nexproto.FriendRequest, 0)
for i := 0; i < len(sliceMap); i++ {
senderPID := uint32(sliceMap[i]["sender_pid"].(int))
rows, err := postgres.Query(`SELECT id, sender_pid, sent_on, expires_on, message, received FROM wiiu.friend_requests WHERE recipient_pid=$1 AND accepted=false AND denied=false`, pid)
if err != nil {
globals.Logger.Critical(err.Error())
return friendRequestsIn
}
for rows.Next() {
var id uint64
var senderPID uint32
var sentOn uint64
var expiresOn uint64
var message string
var received bool
rows.Scan(&id, &senderPID, &sentOn, &expiresOn, &message, &received)
senderUserInforation := GetUserInfoByPID(senderPID)
encodedMiiData := senderUserInforation["mii"].(bson.M)["data"].(string)
@@ -42,18 +46,18 @@ func GetUserFriendRequestsIn(pid uint32) []*nexproto.FriendRequest {
friendRequest.PrincipalInfo.Unknown = 2 // replaying from real server
friendRequest.Message = nexproto.NewFriendRequestMessage()
friendRequest.Message.FriendRequestID = uint64(sliceMap[i]["id"].(int64))
friendRequest.Message.Received = sliceMap[i]["received"].(bool)
friendRequest.Message.FriendRequestID = id
friendRequest.Message.Received = received
friendRequest.Message.Unknown2 = 1
friendRequest.Message.Message = sliceMap[i]["message"].(string)
friendRequest.Message.Message = message
friendRequest.Message.Unknown3 = 0
friendRequest.Message.Unknown4 = ""
friendRequest.Message.GameKey = nexproto.NewGameKey()
friendRequest.Message.GameKey.TitleID = 0
friendRequest.Message.GameKey.TitleVersion = 0
friendRequest.Message.Unknown5 = nex.NewDateTime(134222053376) // idk what this value means but its always this
friendRequest.Message.ExpiresOn = nex.NewDateTime(uint64(sliceMap[i]["expires_on"].(int64)))
friendRequest.SentOn = nex.NewDateTime(uint64(sliceMap[i]["sent_on"].(int64)))
friendRequest.Message.ExpiresOn = nex.NewDateTime(expiresOn)
friendRequest.SentOn = nex.NewDateTime(sentOn)
friendRequestsIn = append(friendRequestsIn, friendRequest)
}

View File

@@ -11,19 +11,22 @@ import (
// Get a users sent friend requests
func GetUserFriendRequestsOut(pid uint32) []*nexproto.FriendRequest {
var sliceMap []map[string]interface{}
var err error
if sliceMap, err = cassandraClusterSession.Query(`SELECT id, recipient_pid, sent_on, expires_on, message, received FROM pretendo_friends.friend_requests WHERE sender_pid=? AND accepted=false AND denied=false ALLOW FILTERING`, pid).Iter().SliceMap(); err != nil {
globals.Logger.Critical(err.Error())
return make([]*nexproto.FriendRequest, 0)
}
friendRequestsOut := make([]*nexproto.FriendRequest, 0)
for i := 0; i < len(sliceMap); i++ {
recipientPID := uint32(sliceMap[i]["recipient_pid"].(int))
rows, err := postgres.Query(`SELECT id, recipient_pid, sent_on, expires_on, message, received FROM wiiu.friend_requests WHERE sender_pid=$1 AND accepted=false AND denied=false`, pid)
if err != nil {
globals.Logger.Critical(err.Error())
return friendRequestsOut
}
for rows.Next() {
var id uint64
var recipientPID uint32
var sentOn uint64
var expiresOn uint64
var message string
var received bool
rows.Scan(&id, &recipientPID, &sentOn, &expiresOn, &message, &received)
recipientUserInforation := GetUserInfoByPID(recipientPID)
encodedMiiData := recipientUserInforation["mii"].(bson.M)["data"].(string)
@@ -43,18 +46,18 @@ func GetUserFriendRequestsOut(pid uint32) []*nexproto.FriendRequest {
friendRequest.PrincipalInfo.Unknown = 2 // replaying from real server
friendRequest.Message = nexproto.NewFriendRequestMessage()
friendRequest.Message.FriendRequestID = uint64(sliceMap[i]["id"].(int64))
friendRequest.Message.Received = sliceMap[i]["received"].(bool)
friendRequest.Message.FriendRequestID = id
friendRequest.Message.Received = received
friendRequest.Message.Unknown2 = 1
friendRequest.Message.Message = sliceMap[i]["message"].(string)
friendRequest.Message.Message = message
friendRequest.Message.Unknown3 = 0
friendRequest.Message.Unknown4 = ""
friendRequest.Message.GameKey = nexproto.NewGameKey()
friendRequest.Message.GameKey.TitleID = 0
friendRequest.Message.GameKey.TitleVersion = 0
friendRequest.Message.Unknown5 = nex.NewDateTime(134222053376) // idk what this value means but its always this
friendRequest.Message.ExpiresOn = nex.NewDateTime(uint64(sliceMap[i]["expires_on"].(int64)))
friendRequest.SentOn = nex.NewDateTime(uint64(sliceMap[i]["sent_on"].(int64)))
friendRequest.Message.ExpiresOn = nex.NewDateTime(expiresOn)
friendRequest.SentOn = nex.NewDateTime(sentOn)
friendRequestsOut = append(friendRequestsOut, friendRequest)
}

View File

@@ -1,11 +1,23 @@
package database
import nexproto "github.com/PretendoNetwork/nex-protocols-go"
import (
"database/sql"
"github.com/PretendoNetwork/friends-secure/globals"
nexproto "github.com/PretendoNetwork/nex-protocols-go"
)
func GetUserPrincipalPreference(pid uint32) *nexproto.PrincipalPreference {
preference := nexproto.NewPrincipalPreference()
_ = cassandraClusterSession.Query(`SELECT show_online, show_current_game, block_friend_requests FROM pretendo_friends.preferences WHERE pid=?`, pid).Scan(&preference.ShowOnlinePresence, &preference.ShowCurrentTitle, &preference.BlockFriendRequests)
err := postgres.QueryRow(`SELECT show_online, show_current_game, block_friend_requests FROM wiiu.user_data WHERE pid=$1`, pid).Scan(&preference.ShowOnlinePresence, &preference.ShowCurrentTitle, &preference.BlockFriendRequests)
if err != nil {
if err == sql.ErrNoRows {
globals.Logger.Warning(err.Error())
} else {
globals.Logger.Critical(err.Error())
}
}
return preference
}

View File

@@ -0,0 +1,65 @@
package database
import "github.com/PretendoNetwork/friends-secure/globals"
func initPostgresWiiU() {
var err error
_, err = postgres.Exec(`CREATE SCHEMA IF NOT EXISTS wiiu`)
if err != nil {
globals.Logger.Critical(err.Error())
return
}
_, err = postgres.Exec(`CREATE TABLE IF NOT EXISTS wiiu.user_data (
pid integer PRIMARY KEY,
show_online boolean,
show_current_game boolean,
block_friend_requests boolean,
comment text,
comment_changed bigint,
last_online bigint
)`)
if err != nil {
globals.Logger.Critical(err.Error())
return
}
_, err = postgres.Exec(`CREATE TABLE IF NOT EXISTS wiiu.friendships (
id bigserial PRIMARY KEY,
user1_pid integer,
user2_pid integer,
date bigint
)`)
if err != nil {
globals.Logger.Critical(err.Error())
return
}
_, err = postgres.Exec(`CREATE TABLE IF NOT EXISTS wiiu.blocks (
id bigserial PRIMARY KEY,
blocker_pid integer,
blocked_pid integer,
date bigint
)`)
if err != nil {
globals.Logger.Critical(err.Error())
return
}
_, err = postgres.Exec(`CREATE TABLE IF NOT EXISTS wiiu.friend_requests (
id bigserial PRIMARY KEY,
sender_pid integer,
recipient_pid integer,
sent_on bigint,
expires_on bigint,
message text,
received boolean,
accepted boolean,
denied boolean
)`)
if err != nil {
globals.Logger.Critical(err.Error())
return
}
}

View File

@@ -1,17 +1,15 @@
package database
import "github.com/gocql/gocql"
import (
"github.com/PretendoNetwork/friends-secure/globals"
)
func IsFriendRequestBlocked(requesterPID uint32, requestedPID uint32) bool {
if err := cassandraClusterSession.Query(`SELECT id FROM pretendo_friends.blocks WHERE blocker_pid=? AND blocked_pid=? LIMIT 1 ALLOW FILTERING`, requestedPID, requesterPID).Scan(); err != nil {
if err == gocql.ErrNotFound {
// Assume no block record was found
return false
}
// TODO: Error handling
var found bool
err := postgres.QueryRow(`SELECT COUNT(*) FROM wiiu.blocks WHERE blocker_pid=$1 AND blocked_pid=$2 LIMIT 1`, requesterPID, requestedPID).Scan(&found)
if err != nil {
globals.Logger.Critical(err.Error())
}
// Assume a block record was found
return true
return found
}

View File

@@ -2,8 +2,16 @@ package database
import "github.com/PretendoNetwork/friends-secure/globals"
func SaveFriendRequest(friendRequestID uint64, senderPID uint32, recipientPID uint32, sentTime uint64, expireTime uint64, message string) {
if err := cassandraClusterSession.Query(`INSERT INTO pretendo_friends.friend_requests (id, sender_pid, recipient_pid, sent_on, expires_on, message, received, accepted, denied) VALUES (?, ?, ?, ?, ?, ?, false, false, false) IF NOT EXISTS`, friendRequestID, senderPID, recipientPID, sentTime, expireTime, message).Exec(); err != nil {
func SaveFriendRequest(senderPID uint32, recipientPID uint32, sentTime uint64, expireTime uint64, message string) uint64 {
var id uint64
err := postgres.QueryRow(`
INSERT INTO wiiu.friend_requests (sender_pid, recipient_pid, sent_on, expires_on, message, received, accepted, denied)
VALUES ($1, $2, $3, $4, $5, false, false, false) RETURNING id`, senderPID, recipientPID, sentTime, expireTime, message).Scan(&id)
if err != nil {
globals.Logger.Critical(err.Error())
return 0
}
return id
}

View File

@@ -3,7 +3,9 @@ package database
import "github.com/PretendoNetwork/friends-secure/globals"
func SetFriendRequestAccepted(friendRequestID uint64) {
if err := cassandraClusterSession.Query(`UPDATE pretendo_friends.friend_requests SET accepted=true WHERE id=?`, friendRequestID).Exec(); err != nil {
_, err := postgres.Exec(`UPDATE wiiu.friend_requests SET accepted=true WHERE id=$1`, friendRequestID)
if err != nil {
globals.Logger.Critical(err.Error())
}
}

View File

@@ -3,7 +3,9 @@ package database
import "github.com/PretendoNetwork/friends-secure/globals"
func SetFriendRequestReceived(friendRequestID uint64) {
if err := cassandraClusterSession.Query(`UPDATE pretendo_friends.friend_requests SET received=true WHERE id=?`, friendRequestID).Exec(); err != nil {
_, err := postgres.Exec(`UPDATE wiiu.friend_requests SET received=true WHERE id=$1`, friendRequestID)
if err != nil {
globals.Logger.Critical(err.Error())
}
}

View File

@@ -9,7 +9,15 @@ import (
func UpdateUserComment(pid uint32, message string) uint64 {
changed := nex.NewDateTime(0).Now()
if err := cassandraClusterSession.Query(`UPDATE pretendo_friends.comments SET message=?, changed=? WHERE pid=?`, message, changed, pid).Exec(); err != nil {
_, err := postgres.Exec(`
INSERT INTO wiiu.user_data (pid, comment, comment_changed)
VALUES ($1, $2, $3)
ON CONFLICT (pid)
DO UPDATE SET
comment = $2,
comment_changed = $3`, pid, message, changed)
if err != nil {
globals.Logger.Critical(err.Error())
}

View File

@@ -6,7 +6,14 @@ import (
)
func UpdateUserLastOnlineTime(pid uint32, lastOnline *nex.DateTime) {
if err := cassandraClusterSession.Query(`UPDATE pretendo_friends.last_online SET time=? WHERE pid=?`, lastOnline.Value(), pid).Exec(); err != nil {
_, err := postgres.Exec(`
INSERT INTO wiiu.user_data (pid, last_online)
VALUES ($1, $2)
ON CONFLICT (pid)
DO UPDATE SET
last_online = $2`, pid, lastOnline.Value())
if err != nil {
globals.Logger.Critical(err.Error())
}
}

View File

@@ -6,11 +6,16 @@ import (
)
func UpdateUserPrincipalPreference(pid uint32, principalPreference *nexproto.PrincipalPreference) {
if err := cassandraClusterSession.Query(`UPDATE pretendo_friends.preferences SET
show_online=?,
show_current_game=?,
block_friend_requests=?
WHERE pid=?`, principalPreference.ShowOnlinePresence, principalPreference.ShowCurrentTitle, principalPreference.BlockFriendRequests, pid).Exec(); err != nil {
_, err := postgres.Exec(`
INSERT INTO wiiu.user_data (pid, show_online, show_current_game, block_friend_requests)
VALUES ($1, $2, $3, $4)
ON CONFLICT (pid)
DO UPDATE SET
show_online = $2,
show_current_game = $3,
block_friend_requests = $4`, pid, principalPreference.ShowOnlinePresence, principalPreference.ShowCurrentTitle, principalPreference.BlockFriendRequests)
if err != nil {
globals.Logger.Critical(err.Error())
}
}