Learning Go - The hard way

Fedrianto Ramadhan -

I have a Task it consists of these points

  • Make Go App using the concept of micro services
  • User has many address, Address belongs to User.
  • Can't use framework
  • Concept MVP, DRY and SOLID
  • Can handle 1 million request using loader.io
  • Centralized Logs
  • Error handling
  • Code Quality

Create Simple Rest Example

I didn't know where to start so I start from here because it has the routing concept, this get me started on learning the concept of GET POST PUT DELETE in go.

go get -v -u github.com/gorilla/mux
package main

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"github.com/gorilla/mux"
)

type Article struct {
	Title   string `json:"Title"`
	Desc    string `json:"Desc"`
	Content string `json:"Content`
}

type Articles []Article

func AllArticles(w http.ResponseWriter, r *http.Request) {
	articles := Articles{
		Article{Title: "Test Title", Desc: "Test Description", Content: "Hello World"},
	}
	fmt.Println("Endpoint : All Article")
	json.NewEncoder(w).Encode(articles)
}

func PostArticles(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "Post Article Endpoint Hit")
}

func homePage(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "Homepage Endpoint Hit")
}

func handleRequest() {
	myRouter := mux.NewRouter().StrictSlash(true)
	myRouter.HandleFunc("/", homePage).Methods("GET")
	myRouter.HandleFunc("/articles", AllArticles).Methods("GET")
	myRouter.HandleFunc("/articles", PostArticles).Methods("POST")
	log.Fatal(http.ListenAndServe(":8081", myRouter))
}

func main() {
	handleRequest()
}

Hot reload

After a while it gets annoying on how you need to always run go run .go so I looked up some Hot reload or live reloading. This came up on the first search of google and it gets the job done so I don't need to check for another hot reload tools.

go get -u github.com/cosmtrek/air

Implementing MVP

I'm begining to understand that you can make packages for everything, so I started to mess around with the concept of microservices, Well first in each package needs to have controller, models and view, with this concept I separate each as a package (don't know if its the best practice) so i have a package of :

  • Main ( this consists of router, boot the app and maybe the home controller if needed)
  • User (using MVC had model and controller, view is still not needed because I'm going to make an API.
  • Address (using MVC with model and controller).

Basic GIN

package main

import (
	"net/http"

	"github.com/gin-gonic/gin"
	"github.com/gosimple/slug"
	"gorm.io/driver/mysql"
	"gorm.io/gorm"
)

type Article struct {
	gorm.Model
	Title string
	Slug string `gorm:"unique_index"`
	Desc string `sql:"type:text;"`
}

var DB *gorm.DB

func main() {

	var err error
	dsn := "test:test123@tcp(127.0.0.1:3306)/test_learningin?charset=utf8mb4&parseTime=True&loc=Local"
	DB , err = gorm.Open(mysql.Open(dsn), &gorm.Config{})
	if err != nil {
		panic("failed to connect database")
	}
	DB.AutoMigrate(Article{})


	r := gin.Default()

	v1 := r.Group("/api/v1/") 
	{
		articles := v1.Group("/article")
		{
			articles.GET("/", getHome)
			articles.GET("/:slug", getArticle)
			articles.POST("/", postArticle)
		}
		
	}
	
	r.Run() 
}

func getHome(c *gin.Context) {

	items := []Article{}
	DB.Find(&items)

	c.JSON( http.StatusOK, gin.H { 
		"status" : "berhasil ke halaman home",
		"data": items,
	})
}

func getArticle(c *gin.Context) {
	slug := c.Param("slug")
	var item Article
	if err := DB.Where("slug = ?", slug).First(&item).Error; err != nil {
		c.JSON(404, gin.H {
			"status" : "error",
			"message" : "data tidak ditemukan",
		})
		c.Abort()
		return
	}

	c.JSON( http.StatusOK, gin.H { 
		"status" : "berhasil",
		"data": item,
	})
}

func postArticle(c *gin.Context) {
	item := Article {
		Title : c.PostForm("title"),
		Desc : c.PostForm("desc"),
		Slug : slug.Make(c.PostForm("title")),
	}

	DB.Create(&item)


	c.JSON( http.StatusOK, gin.H { 
		"status" : "berhasil ngepost",
		"data": item,
	})
}

Restructure code above

// server.go

package main

import (
	"./config"
	"./routes"
	"github.com/gin-gonic/gin"
)

func main() {
	config.InitDB()

	r := gin.Default()

	v1 := r.Group("/api/v1/") 
	{
		articles := v1.Group("/article")
		{
			articles.GET("/", routes.GetHome)
			articles.GET("/:slug", routes.GetArticle)
			articles.POST("/", routes.PostArticle)
		}
		
	}
	
	r.Run() 
}
// config/db.go

package config

import (
	"../models"
	"gorm.io/driver/mysql"
	"gorm.io/gorm"
)

var DB *gorm.DB

func InitDB() {

	var err error
	dsn := "madindo:madindo123@tcp(127.0.0.1:3306)/test_learningin?charset=utf8mb4&parseTime=True&loc=Local"
	DB , err = gorm.Open(mysql.Open(dsn), &gorm.Config{})
	if err != nil {
		panic("failed to connect database")
	}
	DB.AutoMigrate(&models.Article{})
}
// models/article.go

package models

import "gorm.io/gorm"

type Article struct {
	gorm.Model
	Title string
	Slug string `gorm:"unique_index"`
	Desc string `sql:"type:text;"`
}
// routes/article.go

package routes

import (
	"net/http"

	"../config"
	"../models"
	"github.com/gin-gonic/gin"
	"github.com/gosimple/slug"
)

func GetHome(c *gin.Context) {

	items := []models.Article{}
	config.DB.Find(&items)

	c.JSON( http.StatusOK, gin.H { 
		"status" : "berhasil ke halaman home",
		"data": items,
	})
}

func GetArticle(c *gin.Context) {
	slug := c.Param("slug")
	var item models.Article
	if err := config.DB.Where("slug = ?", slug).First(&item).Error; err != nil {
		c.JSON(404, gin.H {
			"status" : "error",
			"message" : "data tidak ditemukan",
		})
		c.Abort()
		return
	}

	c.JSON( http.StatusOK, gin.H { 
		"status" : "berhasil",
		"data": item,
	})
}

func PostArticle(c *gin.Context) {
	item := models.Article {
		Title : c.PostForm("title"),
		Desc : c.PostForm("desc"),
		Slug : slug.Make(c.PostForm("title")),
	}

	config.DB.Create(&item)


	c.JSON( http.StatusOK, gin.H { 
		"status" : "berhasil ngepost",
		"data": item,
	})
}

Social Authentication

https://github.com/danilopolani/gocialite - use this for package social authentication but now it's not maintain, still haven't found another package like this.

https://github.com/subosito/gotenv - use this package to enable .env

// routes/auth.go

package routes

import (
	"fmt"
	"net/http"
	"os"

	"../config"
	"github.com/gin-gonic/gin"
)

// Redirect to correct oAuth URL
func RedirectHandler(c *gin.Context) {
	// Retrieve provider from route
	provider := c.Param("provider")

	// In this case we use a map to store our secrets, but you can use dotenv or your framework configuration
	// for example, in revel you could use revel.Config.StringDefault(provider + "_clientID", "") etc.
	providerSecrets := map[string]map[string]string{
		"github": {
			"clientID":     os.Getenv("CLIENT_ID_GH"),
			"clientSecret": os.Getenv("CLIENT_SECRET_GH"),
			"redirectURL":  os.Getenv("AUTH_REDIRECT_URL") + "/github/callback",
		},
	}

	providerScopes := map[string][]string{
		"github":   []string{"public_repo"},
	}

	providerData := providerSecrets[provider]
	actualScopes := providerScopes[provider]
	authURL, err := config.Gocial.New().
		Driver(provider).
		Scopes(actualScopes).
		Redirect(
			providerData["clientID"],
			providerData["clientSecret"],
			providerData["redirectURL"],
		)

	// Check for errors (usually driver not valid)
	if err != nil {
		c.Writer.Write([]byte("Error: " + err.Error()))
		return
	}

	// Redirect with authURL
	c.Redirect(http.StatusFound, authURL)
}
// Handle callback of provider
func CallbackHandler(c *gin.Context) {
	state := c.Query("state")
	code := c.Query("code")
	provider := c.Param("provider")

	user, token, err := config.Gocial.Handle(state, code)
	if err != nil {
		c.Writer.Write([]byte("Error: " + err.Error()))
		return
	}

	var newUser = getOrRegisterUser(provider, user)

	c.JSON(200, gin.H{
		"data": newUser,
		"token": token,
		"message": "Successfully login",
	});
	
}

func getOrRegisterUser(provider string, user *structs.User) models.User{
	var userData models.User

	config.DB.Where("provider = ? and social_id = ?", provider, user.ID).First(&userData)

	if userData.ID == 0 {
		newUser := models.User {
			FullName: user.FullName,
			Email: user.Email,
			SocialID: user.ID,
			Provider: provider,
			Avatar: user.Avatar,
		}
		config.DB.Create(&newUser)
		return newUser
	} else {
		return userData
	}
}
// config/auth.go

package config

import (
	"gopkg.in/danilopolani/gocialite.v1"
)

var Gocial = gocialite.NewDispatcher()
v1 := r.Group("/api/v1/") 
	{

		//add this
		v1.GET("auth/:provider", routes.RedirectHandler)
		v1.GET("auth/:provider/callback", routes.CallbackHandler)
		//end

		articles := v1.Group("/article")
		{
			articles.GET("/", routes.GetHome)
			articles.GET("/:slug", routes.GetArticle)
			articles.POST("/", routes.PostArticle)
		}
		
	}

Implementing JWT

go get "https://github.com/dgrijalva/jwt-go"

Create Token

// routes/auth.go

func CallbackHandler(c *gin.Context) {
	***
	var newUser = getOrRegisterUser(provider, user)
	var jwtToken = createToken(&newUser)
	***
}


func createToken(user *models.User) string {
	token := jwt.NewWithClaims(jwt.SigningMethodHS256, jwt.MapClaims{
		"user_id": user.ID,
		"user_role": user.Role,
		"exp": json.Number(strconv.FormatInt(time.Now().Add(time.Hour*time.Duration(1)).Unix(), 10)),
		"iat": json.Number(strconv.FormatInt(time.Now().Unix(), 10)),
	})

	tokenString, err := token.SignedString([]byte(JWT_SECRET))

	if err != nil {
		fmt.Println(err)
	}
	return tokenString
}

Read Token

v1 := r.Group("/api/v1/") 
{
	****
	v1.GET("check", middleware.IsAuth(), routes.CheckToken)
	****
}
// middleware/auth.go

package middleware

import (
	"fmt"
	"strings"

	"github.com/dgrijalva/jwt-go"
	"github.com/gin-gonic/gin"
)

var JWT_SECRET = "SUPER_SECRET"

func IsAuth() gin.HandlerFunc {
	return checkJWT()
}

func checkJWT() gin.HandlerFunc {
	return func(c *gin.Context) {
		authHeader := c.Request.Header.Get("Authorization")
		bearerToken := strings.Split(authHeader," ")

		token, err := jwt.Parse(bearerToken[1], func(token *jwt.Token) (interface{}, error) {
			return []byte(JWT_SECRET), nil
		})

		if err == nil && token.Valid {
			claims := token.Claims.(jwt.MapClaims)
			fmt.Println(claims["user_id"])
			fmt.Println(claims["user_role"])
		} else {
			fmt.Println("KO")
		}
	}
	
}
// routes/auth.go

func CheckToken(c *gin.Context) {
	c.JSON(200, gin.H{"msg": "success"})
}

Using ORM

package controllers
import (
	"encoding/json"
	"net/http"
	"fmt"
	"github.com/jinzhu/gorm"
	_ "github.com/jinzhu/gorm/dialects/sqlite"
	"ownpackage/models"
	"github.com/gorilla/mux"
)

var db *gorm.DB
var err error

func UserIndex (response http.ResponseWriter, request *http.Request) {
	db, err = gorm.Open("sqlite3", "database/harukaedu.db")
	if err != nil {
		fmt.Println(err.Error())
		panic("Failed to open");
	}
	defer db.Close()
	
	var users [] models.User
	db.Find(&users)
	json.NewEncoder(response).Encode(users)
}

func UserStore (response http.ResponseWriter, request *http.Request) {
	
	db, err = gorm.Open("sqlite3", "database/harukaedu.db")
	if err != nil {
		fmt.Println(err.Error())
		panic("Failed to open");
	}
	defer db.Close()

	name := request.FormValue("name")
	email := request.FormValue("email")

	db.Create(&models.User{Name: name, Email: email}) 
	fmt.Fprintf(response, "User successfully Created")
}

func UserUpdate (response http.ResponseWriter, request *http.Request) {
	
	db, err = gorm.Open("sqlite3", "database/harukaedu.db")
	if err != nil {
		fmt.Println(err.Error())
		panic("Failed to open");
	}
	defer db.Close()
	
	vars := mux.Vars(request)
	id := vars["id"]
	name := request.FormValue("name")
	email := request.FormValue("email")

	var user models.User
	db.First(&user)
	db.Where("ID = ?", id).Find(&user)

	user.Name = name
	user.Email = email

	db.Save(&user)
	fmt.Fprintf(response, "User successfully Updated")

}

func UserDelete (response http.ResponseWriter, request *http.Request) {
	
	db, err = gorm.Open("sqlite3", "database/harukaedu.db")
	if err != nil {
		fmt.Println(err.Error())
		panic("Failed to open");
	}
	defer db.Close()
	
	vars := mux.Vars(request)
	id := vars["id"]

	var user models.User
	db.Where("ID = ?", id).Find(&user)
	db.Delete(&user)
	fmt.Fprintf(response, "User successfully Deleted")
}

Debug mode GORM

db.LogMode(true)
db.Debug()

Single Naming in table

Lets say you have CategoryProduct

type CategoryProduct struct {
    gorm.Model
    CategoryProductID   uint        `gorm:"primaryKey"`
    CategoryID          uint        `json:"category_id"`
    ProductID           uint        `json:"product_id"`
    CreatedAt                       time.Time
    UpdatedAt                       time.Time
}

add this below

type Tabler interface {
    TableName() string
}

// TableName overrides the table name used by User to `profiles`
func (CategoryProduct) TableName() string {
    return "category_product"
}

Eager Loading has many in GO

Fetch Data using One To Many Relationship in GORM
var users [] models.User
db.Preload("Address").Find(&users)

Error Handling

Before jumping to Error handling we need to learn what are the context of error in go.
Context matters :

  • Is your program is CLI tools
  • Is your program Library
  • is your program a long time running system
  • Who is consuming your program? And How
package errors
import (
	"fmt"
	"string"
)

func ErrorConnection(err error) {
	if err != nil {
		fmt.Println(err.Error())
		panic("Failed to open");
	}
}

Logs

https://www.scalyr.com/blog/getting-started-quickly-with-go-logging

package logs

import (
	"log"
	"os"
)

func Logging(msgtype string, msg string) {
	file, err := os.OpenFile("logs/all.log", os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644)
	if err != nil {
		log.Fatal(err)
	}

	defer file.Close()

	log.SetOutput(file)
	log.Print( " [ " + msgtype + " ] " + msg)
}

Dump and Die Go Version

package main

import (
    "fmt"
    "os"
)


func main() {
    fmt.Println("Start")
    os.Exit(1)
    fmt.Println("End")
}

Cleaning Go.Sum & Go.Mod

after removing the package from go.sum then run this

go mod tidy

Handling 1mio Request

I will be using loader.io for handle request but when I tried 1 mio, it limits to 10k only

Oh well, lets try with 10k users the scenario is to list all users

10k Client Per Test

10k Client Per Second

Lol... error threshold... lets divide by 2 so 5k per sec ... i'm using single core server, i see some surges when trying 10k per min

it crashes my app or database - unable to open database file: too many open files

oh wow it closes as well 5k... lets divide by 2 again 2.5k

Shocking... btw i'm using sqlite

lets try 1000

it works... so ( newbie deduction ) to get this working I think I need to use concurrency concept ( which I haven't learned yet ) and go routine. To get 1mio request I'll be needing a test per second and when my limit is 10k, I think I need to get about 16666 client per request for one minutes ( 10k / 60 minutes )