[{"content":"","date":"30 August 2026","externalUrl":null,"permalink":"/","section":"/home/fel1x","summary":"","title":"/home/fel1x","type":"page"},{"content":"","date":"30 August 2026","externalUrl":null,"permalink":"/tags/devops/","section":"Tags","summary":"","title":"devops","type":"tags"},{"content":"","date":"30 August 2026","externalUrl":null,"permalink":"/tags/devops/automation/","section":"Tags","summary":"","title":"devops/automation","type":"tags"},{"content":"","date":"30 August 2026","externalUrl":null,"permalink":"/tags/devops/automation/argo-cd/","section":"Tags","summary":"","title":"devops/automation/argo-cd","type":"tags"},{"content":"","date":"30 August 2026","externalUrl":null,"permalink":"/tags/devops/automation/github-actions/","section":"Tags","summary":"","title":"devops/automation/github-actions","type":"tags"},{"content":"","date":"30 August 2026","externalUrl":null,"permalink":"/tags/devops/automation/opentofu/","section":"Tags","summary":"","title":"devops/automation/opentofu","type":"tags"},{"content":"","date":"30 August 2026","externalUrl":null,"permalink":"/tags/devops/cloud/","section":"Tags","summary":"","title":"devops/cloud","type":"tags"},{"content":"","date":"30 August 2026","externalUrl":null,"permalink":"/tags/devops/cloud/aws/","section":"Tags","summary":"","title":"devops/cloud/aws","type":"tags"},{"content":"","date":"30 August 2026","externalUrl":null,"permalink":"/tags/devops/cloud/proxmox/","section":"Tags","summary":"","title":"devops/cloud/proxmox","type":"tags"},{"content":"","date":"30 August 2026","externalUrl":null,"permalink":"/tags/devops/container/","section":"Tags","summary":"","title":"devops/container","type":"tags"},{"content":"","date":"30 August 2026","externalUrl":null,"permalink":"/tags/devops/container/kubernetes/","section":"Tags","summary":"","title":"devops/container/kubernetes","type":"tags"},{"content":"","date":"30 August 2026","externalUrl":null,"permalink":"/tags/devops/security/","section":"Tags","summary":"","title":"devops/security","type":"tags"},{"content":"","date":"30 August 2026","externalUrl":null,"permalink":"/tags/devops/security/external-secrets/","section":"Tags","summary":"","title":"devops/security/external-secrets","type":"tags"},{"content":"","date":"30 August 2026","externalUrl":null,"permalink":"/tags/network/","section":"Tags","summary":"","title":"network","type":"tags"},{"content":"","date":"30 August 2026","externalUrl":null,"permalink":"/tags/network/self-hosted/","section":"Tags","summary":"","title":"network/self-hosted","type":"tags"},{"content":"","date":"30 August 2026","externalUrl":null,"permalink":"/tags/os/","section":"Tags","summary":"","title":"os","type":"tags"},{"content":"","date":"30 August 2026","externalUrl":null,"permalink":"/tags/os/linux/","section":"Tags","summary":"","title":"os/linux","type":"tags"},{"content":"","date":"30 August 2026","externalUrl":null,"permalink":"/projects/","section":"Projects","summary":"","title":"Projects","type":"projects"},{"content":"","date":"30 August 2026","externalUrl":null,"permalink":"/tags/","section":"Tags","summary":"","title":"Tags","type":"tags"},{"content":"","date":"30 August 2026","externalUrl":"https://github.com/phuchoang2603/talos-proxmox","permalink":"/projects/talos-proxmox/","section":"Projects","summary":"This project automates independent development and production Talos Kubernetes environments on Proxmox using OpenTofu and GitHub Actions, with optional stateless AWS workers connected through KubeSpan. Each cluster runs its own Argo CD for GitOps application management, while Doppler and External Secrets supply credentials and persistent workloads stay on Proxmox.","title":"Talos on Proxmox – Hybrid Kubernetes with AWS Burst Workers","type":"projects"},{"content":"","date":"29 March 2026","externalUrl":null,"permalink":"/tags/data/","section":"Tags","summary":"","title":"data","type":"tags"},{"content":"","date":"29 March 2026","externalUrl":null,"permalink":"/tags/data/database/","section":"Tags","summary":"","title":"data/database","type":"tags"},{"content":"","date":"29 March 2026","externalUrl":null,"permalink":"/tags/devops/monitoring/","section":"Tags","summary":"","title":"devops/monitoring","type":"tags"},{"content":"","date":"29 March 2026","externalUrl":null,"permalink":"/tags/devops/monitoring/tracing/","section":"Tags","summary":"","title":"devops/monitoring/tracing","type":"tags"},{"content":"","date":"29 March 2026","externalUrl":null,"permalink":"/tags/network/api/","section":"Tags","summary":"","title":"network/api","type":"tags"},{"content":"","date":"29 March 2026","externalUrl":null,"permalink":"/tags/programming/","section":"Tags","summary":"","title":"programming","type":"tags"},{"content":"","date":"29 March 2026","externalUrl":null,"permalink":"/tags/programming/lang/","section":"Tags","summary":"","title":"programming/lang","type":"tags"},{"content":"","date":"29 March 2026","externalUrl":null,"permalink":"/tags/programming/lang/go/","section":"Tags","summary":"","title":"programming/lang/go","type":"tags"},{"content":"","date":"29 March 2026","externalUrl":"https://github.com/phuchoang2603/refurbished-marketplace","permalink":"/projects/refurbished-marketplace/","section":"Projects","summary":"This project explores distributed Go microservices through a refurbished-goods marketplace, with a server-rendered storefront, catalog search, inventory reservations, and merchant-scoped checkout. It combines gRPC APIs, Kafka and Debezium messaging, a durable checkout saga, and Argo CD deployments on Talos Kubernetes with Istio ambient mTLS and OpenTelemetry observability.","title":"Refurbished Marketplace – Distributed Go Microservices","type":"projects"},{"content":"","date":"29 March 2026","externalUrl":null,"permalink":"/tags/system-design/","section":"Tags","summary":"","title":"system-design","type":"tags"},{"content":"","date":"29 March 2026","externalUrl":null,"permalink":"/tags/system-design/cache/","section":"Tags","summary":"","title":"system-design/cache","type":"tags"},{"content":"","date":"29 March 2026","externalUrl":null,"permalink":"/tags/system-design/event-driven/","section":"Tags","summary":"","title":"system-design/event-driven","type":"tags"},{"content":"","date":"29 March 2026","externalUrl":null,"permalink":"/tags/system-design/messaging/","section":"Tags","summary":"","title":"system-design/messaging","type":"tags"},{"content":"","date":"29 March 2026","externalUrl":null,"permalink":"/tags/system-design/microservice/","section":"Tags","summary":"","title":"system-design/microservice","type":"tags"},{"content":"","date":"24 November 2025","externalUrl":null,"permalink":"/tags/devops/security/hashicorp-vault/","section":"Tags","summary":"","title":"devops/security/hashicorp-vault","type":"tags"},{"content":"","date":"24 November 2025","externalUrl":null,"permalink":"/series/hashicorp-vault/","section":"Series","summary":"","title":"Hashicorp Vault","type":"series"},{"content":"","date":"24 November 2025","externalUrl":null,"permalink":"/posts/","section":"Posts","summary":"","title":"Posts","type":"posts"},{"content":"","date":"24 November 2025","externalUrl":null,"permalink":"/series/","section":"Series","summary":"","title":"Series","type":"series"},{"content":"This is Part 4 of the Hashicorp Vault series. In Part 1, we bootstrapped Vault. In Part 2, we built our CI/CD pipeline. In Part 3, we finally killed the static kubeconfig.\nWe\u0026rsquo;ve secured our CI runners and our human kubectl access. But what about the apps themselves? We\u0026rsquo;re on the final, most important step: how do applications running inside the cluster securely fetch their secrets?\nThe Last-Mile Problem: Secrets in Your Pods # So far, our \u0026ldquo;single source of truth\u0026rdquo; for secrets is Vault. But our applications don\u0026rsquo;t know how to talk to Vault. They just know how to read a standard Kubernetes Secret.\nThe \u0026ldquo;easy\u0026rdquo; (and wrong) way to solve this is to just do it manually:\nkubectl create secret generic db-credentials \\ --from-literal=username=myapp \\ --from-literal=password=super-secret-password The moment you run this, you\u0026rsquo;ve undone all the hard work from the last three posts.\nYour \u0026ldquo;Single Source of Truth\u0026rdquo; is now a lie. You have a password in Vault and a copy of it base64-encoded in etcd. Which one is right?\nYou just broke your automation. When that password rotates in Vault (and it should!), your app will break. You\u0026rsquo;ll get paged at 3 AM to go manually run kubectl again.\nYou have no audit trail. Who created this secret? Where did \u0026ldquo;super-secret-password\u0026rdquo; come from?\nIt\u0026rsquo;s a static, long-lived key. A compromised pod now has that password forever.\nWe\u0026rsquo;ve spent three articles building a beautiful Zero Trust secrets engine. We\u0026rsquo;re not going to fall at the final hurdle.\nThe Solution: External Secrets Operator (ESO) # The solution is a little robot that lives in your cluster called the External Secrets Operator (ESO).\nIt\u0026rsquo;s a Kubernetes operator that does one job, and it does it perfectly. When you create a new YAML file called an ExternalSecret, ESO would automatically logs into Vault, fetches the data, and creates a native Kubernetes Secret for you. After all of that, your pod can now mounts that native Secret just like it always has.\nImage from official docs https://external-secrets.io/latest/\nIf you update the password in Vault, ESO sees the change, updates the native Secret, and your pod (if it\u0026rsquo;s configured to) reloads the new credential.\nThis is the holy grail:\nVault is the single source of truth.\nRotation is automatic.\nDevelopers get the ease of native K8s Secrets.\nApplications don\u0026rsquo;t even know Vault exists.\nBut this all hinges on one critical question\u0026hellip; how does ESO log into Vault?\nThe Real Magic: How ESO Authenticates to Vault # We\u0026rsquo;re not going to give ESO a static Vault token. That\u0026rsquo;s just as bad as a static password. Instead, we\u0026rsquo;re going to use Vault\u0026rsquo;s built-in Kubernetes Auth Method. This is a beautiful, token-for-token exchange. And there are two ways Vault can ask the K8s API \u0026ldquo;is this token legit?\u0026rdquo;:\nReviewer JWT Mode (Old Way): You create a separate, long-lived K8s token (a \u0026ldquo;reviewer token\u0026rdquo;) and give it to Vault. Vault uses this token to validate the other token. It\u0026rsquo;s secure, but now you have a long-lived key to manage.\nClient JWT Mode (New, Better Way): This is the magic. We tell Vault: \u0026ldquo;You don\u0026rsquo;t need a special reviewer token. Just use the same token the client (ESO) gave you to validate itself.\u0026rdquo; -\u0026gt; Yes, please.\nHow is this possible? It works because we\u0026rsquo;re going to give the ESO ServiceAccount one special, extra permission: system:auth-delegator. This built-in K8s role gives it just enough power to ask the TokenReview API, \u0026ldquo;Hey, am I me?\u0026rdquo;\nThis is the entire authentication flow:\nImage created using Nano Banana Pro\nYou create a new YAML file called an ExternalSecret.\nThis file tells ESO, \u0026ldquo;Hey, go look at kv/dev/postgres in Vault.\u0026rdquo;\nESO sends its own ServiceAccount JWT to Vault.\nVault (which can\u0026rsquo;t validate the token itself) turns around and sends that same JWT to the K8s TokenReview API.\nBecause the ESO ServiceAccount has the system:auth-delegator role, the K8s API allows this self-check.\nThe API says, \u0026ldquo;Yep, that token is valid. It belongs to system:serviceaccount:external-secrets:external-secrets.\u0026rdquo;\nVault checks that identity against its external-secrets role.\nVault issues a short-lived Vault token, and ESO gets to work.\nESO automatically logs into Vault, fetches the data, creates a native Kubernetes Secret for you.\nYour pod just mounts that native Secret like it always has.\nStep 1: Configure Vault to Trust Kubernetes (Terraform) # Before any of this works, we need to tell Vault to expect these logins from Kubernetes. This is a one-time setup in our terraform-admin project.\nWe\u0026rsquo;re essentially creating a new \u0026ldquo;door\u0026rdquo; for our K8s cluster to use.\nFile: terraform-admin/modules/vault-kubernetes-auth/main.tf\n# Fetch VIP from Vault KV (stored at kv/{env}/ip) data \u0026#34;vault_generic_secret\u0026#34; \u0026#34;ip\u0026#34; { path = \u0026#34;kv/${var.env}/ip\u0026#34; } # Enable Kubernetes auth method for the environment resource \u0026#34;vault_auth_backend\u0026#34; \u0026#34;kubernetes\u0026#34; { type = \u0026#34;kubernetes\u0026#34; path = \u0026#34;${var.env}-kubernetes\u0026#34; description = \u0026#34;Kubernetes auth backend for ${var.env} environment\u0026#34; } # Configure the Kubernetes auth method using the VIP from Vault resource \u0026#34;vault_kubernetes_auth_backend_config\u0026#34; \u0026#34;kubernetes\u0026#34; { backend = vault_auth_backend.kubernetes.path # Use VIP from Vault KV store (reads the \u0026#39;vip\u0026#39; key from kv/{env}/data/ip) # External Vault connects to Kubernetes API via the VIP on port 6443 kubernetes_host = \u0026#34;https://${data.vault_generic_secret.ip.data[\u0026#34;vip\u0026#34;]}:6443\u0026#34; # Enable client JWT mode - Vault uses the client\u0026#39;s JWT for TokenReview disable_local_ca_jwt = true } # Create a policy for External Secrets Operator resource \u0026#34;vault_policy\u0026#34; \u0026#34;external_secrets\u0026#34; { name = \u0026#34;${var.env}-external-secrets-policy\u0026#34; policy = \u0026lt;\u0026lt;-EOT # Allow reading all secrets in the environment\u0026#39;s KV path path \u0026#34;kv/${var.env}/data/*\u0026#34; { capabilities = [\u0026#34;read\u0026#34;, \u0026#34;list\u0026#34;] } # Allow listing the KV metadata path \u0026#34;kv/${var.env}/metadata/*\u0026#34; { capabilities = [\u0026#34;list\u0026#34;] } EOT } # Create a role for External Secrets Operator resource \u0026#34;vault_kubernetes_auth_backend_role\u0026#34; \u0026#34;external_secrets\u0026#34; { backend = vault_auth_backend.kubernetes.path role_name = \u0026#34;external-secrets\u0026#34; # Bind to the external-secrets service account in external-secrets namespace bound_service_account_names = [\u0026#34;external-secrets\u0026#34;] bound_service_account_namespaces = [\u0026#34;external-secrets\u0026#34;] # Assign the policy token_policies = [vault_policy.external_secrets.name] # Token TTL settings token_ttl = 3600 # 1 hour token_max_ttl = 86400 # 24 hours } Breaking this down:\nFetch VIP: We first need to tell Vault how to find our K8s cluster. We data source the vip (e.g., 10.69.1.110) that our terraform-provision job saved to Vault in Part 2.\nEnable Auth Backend: We enable the kubernetes auth method at a unique path for each environment (dev-kubernetes, prod-kubernetes). This is critical for isolation.\nConfigure Auth Backend: We set kubernetes_host to our VIP. But the most critical setting is disable_local_ca_jwt: true. This is the little toggle that officially enables the Client JWT Mode.\nCreate Policy: We write a policy (dev-external-secrets-policy) that gives ESO read-only access to kv/dev/data/*. This is the only thing this identity will be allowed to do.\nCreate Role: We tie it all together. We create a role named external-secrets that says: \u0026ldquo;Any pod authenticating with the ServiceAccount named external-secrets in the external-secrets namespace\u0026hellip; will get the dev-external-secrets-policy.\u0026rdquo;\nAfter running terraform apply, Vault now has this new \u0026ldquo;door\u0026rdquo; wide open, waiting for our K8s cluster to come knocking.\nStep 2: Deploy ESO via Ansible # Now we flip over to our Ansible playbook. It\u0026rsquo;s time to install ESO and give it the \u0026ldquo;magic key\u0026rdquo; it needs to authenticate.\nHelm Configuration # First, we add external-secrets to our list of Helm apps to deploy. This is standard stuff. It installs the operator, its CRDs (so we can create ExternalSecret resources), and two crucial extra manifests.\nFile: ansible/inventory/group_vars/all/helm.yaml\nhelm_applications: # ... other applications ... # external-secrets: Integrate external secret management systems - name: external-secrets chart: external-secrets version: v0.19.2 repo: https://charts.external-secrets.io namespace: external-secrets create_namespace: true values_content: | installCRDs: true metrics: service: enabled: true serviceMonitor: enabled: true additional_manifests: - external-secrets-rbac - external-secrets-cluster-store RBAC Configuration # This is the first critical piece. We deploy a ClusterRoleBinding that gives our external-secrets ServiceAccount\u0026hellip; the system:auth-delegator role.\nFile: ansible/roles/deploy_helm_apps/templates/additional/external-secrets-rbac.yaml.j2\n--- # ClusterRoleBinding to grant External Secrets Operator the system:auth-delegator role # This allows Vault to use the ESO\u0026#39;s ServiceAccount JWT for TokenReview API calls # Required for Vault Kubernetes auth in client JWT mode apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: external-secrets-auth-delegator roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: system:auth-delegator subjects: - kind: ServiceAccount name: external-secrets namespace: external-secrets Without this ClusterRoleBinding, when Vault tries to call the TokenReview API using ESO\u0026rsquo;s JWT, Kubernetes would return:\nError: User \u0026#34;system:serviceaccount:external-secrets:external-secrets\u0026#34; cannot create resource \u0026#34;tokenreviews\u0026#34; in API group \u0026#34;authentication.k8s.io\u0026#34; The system:auth-delegator role grants permission to create TokenReview requests, which is exactly what Vault needs to do in Client JWT mode.\nClusterSecretStore Configuration # This is the second critical piece. We create a ClusterSecretStore (which is a Cluster wide, not namespace specific, store).\nFile: ansible/roles/deploy_helm_apps/templates/additional/external-secrets-cluster-store.yaml.j2\n--- # ClusterSecretStore for Vault using Kubernetes auth apiVersion: external-secrets.io/v1 kind: ClusterSecretStore metadata: name: vault-backend spec: provider: vault: server: \u0026#34;{{ vault_addr }}\u0026#34; path: \u0026#34;kv/{{ env }}\u0026#34; version: \u0026#34;v2\u0026#34; auth: kubernetes: mountPath: \u0026#34;{{ env }}-kubernetes\u0026#34; role: \u0026#34;external-secrets\u0026#34; serviceAccountRef: name: \u0026#34;external-secrets\u0026#34; namespace: \u0026#34;external-secrets\u0026#34; This YAML is the \u0026ldquo;config file\u0026rdquo; for ESO. It tells ESO:\nWhere is Vault? (server: \u0026quot;{{ vault_addr }}\u0026quot;)\nWhat path should I use? (path: \u0026quot;kv/{{ env }}\u0026quot;)\nHow do I log in? (Using the kubernetes auth method at the {{ env }}-kubernetes mount path, with the external-secrets role.)\nStep 3: Solving the CA Cert Chicken-and-Egg # We\u0026rsquo;re almost done, but there\u0026rsquo;s one last critical issue that we need to address.\nVault is outside our cluster.\nThe K8s API server is inside our cluster and uses a TLS certificate.\nVault (running terraform-admin) generates that TLS certificate during the RKE2 bootstrap.\nWhen Vault tries to validate a token (Step 3 of the auth flow), it makes an HTTPS call to the K8s API. How does Vault trust the K8s API\u0026rsquo;s certificate?\nIt can\u0026rsquo;t. Not out of the box. We have to give Vault the Kubernetes CA certificate.\nBut the CA certificate isn\u0026rsquo;t created until Ansible runs\u0026hellip; and our GitHub Actions workflow needs to update Vault\u0026hellip; after Ansible runs.\nThis means we need a multi-step \u0026ldquo;handshake\u0026rdquo; in our pipeline. Here\u0026rsquo;s how we solve it:\n(GitHub Action) The ansible-bootstrap job runs.\n(Ansible) RKE2 is installed, and the K8s CA cert (k8s-ca.crt) is generated on the control-plane nodes.\n(Ansible) ESO is deployed via Helm.\n(Ansible) As a final task, Ansible kubectls into the cluster, fetches the base64-encoded CA cert, decodes it, and saves it as a file (k8s-ca.crt) on the GitHub runner\u0026rsquo;s local filesystem.\n# ... (earlier roles: download_rke2, add_server, add_agent) ... # Deploy applications - name: Deploy applications hosts: servers gather_facts: true run_once: true roles: - role: apply_kube_vip - role: deploy_helm_apps # Export Kubernetes CA certificate for Vault configuration - name: Export Kubernetes credentials for Vault hosts: servers[0] gather_facts: false tasks: - name: Get Kubernetes CA certificate ansible.builtin.command: cmd: kubectl config view --raw --minify --flatten -o jsonpath=\u0026#39;{.clusters[0].cluster.certificate-authority-data}\u0026#39; environment: KUBECONFIG: /etc/rancher/rke2/rke2.yaml register: k8s_ca_cert_b64 changed_when: false - name: Decode and save Kubernetes CA certificate locally ansible.builtin.copy: content: \u0026#34;{{ k8s_ca_cert_b64.stdout | b64decode }}\u0026#34; dest: \u0026#34;{{ playbook_dir }}/k8s-ca.crt\u0026#34; mode: \u0026#34;0644\u0026#34; delegate_to: localhost (GitHub Action) The ansible-bootstrap job finishes -\u0026gt; A new, separate step in our workflow (Configure Vault Kubernetes Auth) now runs. - name: Run playbook env: OIDC_ISSUER_URL: ${{ vars.VAULT_ADDR }}/v1/identity/oidc/provider/${{ vars.ENV_NAME }} VAULT_ADDR: ${{ vars.VAULT_ADDR }} run: ansible-playbook -e \u0026#34;env=${{ vars.ENV_NAME }}\u0026#34; --private-key runner_key site.yaml - name: Configure Vault Kubernetes Auth env: VAULT_ADDR: ${{ vars.VAULT_ADDR }} VAULT_TOKEN: ${{ steps.vault_auth.outputs.vault_token }} ENV_NAME: ${{ vars.ENV_NAME }} run: | # Configure Vault Kubernetes auth backend with client JWT mode # ESO will use its own ServiceAccount token for both auth and TokenReview vault write auth/${ENV_NAME}-kubernetes/config \\ kubernetes_host=\u0026#34;https://${{ env.IP_VIP }}:6443\u0026#34; \\ kubernetes_ca_cert=@k8s-ca.crt \\ disable_local_ca_jwt=true (GitHub Action) This step uses the Vault token it already has, and runs a vault write command\u0026hellip; uploading the k8s-ca.crt file that Ansible just saved. This is the final piece of glue. We\u0026rsquo;re using our CI runner as a temporary \u0026ldquo;mule\u0026rdquo; to carry the dynamically-generated CA cert from the K8s cluster back to Vault.\nIt\u0026rsquo;s a beautiful bit of automation, and it solves the chicken-and-egg problem perfectly.\nUsing External Secrets # Okay. The platform is built. The engine is running. Now, how does a developer on my team actually use this?\nLet\u0026rsquo;s say they need database credentials, which I\u0026rsquo;ve stored in Vault at kv/dev/postgres.\nAll they have to do is add this ExternalSecret YAML to their application\u0026rsquo;s Helm chart or GitOps repo:\napiVersion: external-secrets.io/v1 kind: ExternalSecret metadata: name: postgres-credentials namespace: myapp spec: # Refresh the secret every hour refreshInterval: 1h # Reference the ClusterSecretStore we created secretStoreRef: kind: ClusterSecretStore name: vault-backend # The target Kubernetes Secret to create/update target: name: postgres-credentials creationPolicy: Owner # Map Vault secret fields to Kubernetes Secret keys data: - secretKey: host remoteRef: key: postgres # kv/dev/data/postgres property: host # Field name in Vault secret - secretKey: port remoteRef: key: postgres property: port - secretKey: username remoteRef: key: postgres property: username - secretKey: password remoteRef: key: postgres property: password - secretKey: database remoteRef: key: postgres property: database When they kubectl apply this, here\u0026rsquo;s what happens:\nESO sees the new ExternalSecret resource.\nIt checks the ClusterSecretStore (our vault-backend).\nIt authenticates to Vault (using the whole Client JWT flow).\nIt fetches the data from kv/dev/data/postgres.\nIt creates a new, native Kubernetes Secret called postgres-credentials in the myapp namespace.\napiVersion: v1 kind: Secret metadata: name: postgres-credentials namespace: myapp type: Opaque data: host: cG9zdGdyZXMuZGVmYXVsdC5zdmMuY2x1c3Rlci5sb2NhbA== # base64 port: NTQzMg== username: bXlhcHA= password: c3VwZXItc2VjcmV0LXBhc3N3b3Jk database: bXlhcHBfZGI= Their pod deployment YAML is now\u0026hellip; completely standard. It doesn\u0026rsquo;t know (or care) about Vault, ESO, or anything else. It just mounts the native Secret that ESO created for it.\napiVersion: v1 kind: Pod metadata: name: myapp namespace: myapp spec: containers: - name: app image: myapp:latest env: - name: DB_HOST valueFrom: secretKeyRef: name: postgres-credentials key: host - name: DB_PORT valueFrom: secretKeyRef: name: postgres-credentials key: port - name: DB_USERNAME valueFrom: secretKeyRef: name: postgres-credentials key: username - name: DB_PASSWORD valueFrom: secretKeyRef: name: postgres-credentials key: password - name: DB_DATABASE valueFrom: secretKeyRef: name: postgres-credentials key: database And that\u0026rsquo;s it. We\u0026rsquo;ve done it. We\u0026rsquo;ve built a system where developers can consume secrets from a single source of truth in a secure, automated, and K8s-native way.\nSummary # In this part, we closed the final loop in our Zero Trust secret management story.\nWe configured Vault\u0026rsquo;s Kubernetes Auth to trust our cluster\u0026rsquo;s ServiceAccounts.\nWe used the Client JWT Mode to avoid managing any long-lived tokens.\nWe deployed the External Secrets Operator with the magic system:auth-delegator permission.\nWe solved the CA cert chicken-and-egg problem with a clever Ansible -\u0026gt; GitHub Action handshake.\nWe provided a dead-simple workflow for developers to consume secrets.\nNext up: We have the infrastructure, we have the secrets\u0026hellip; now we need to deploy the apps. This marks the beginning of the GitOps series coming up.\n","date":"24 November 2025","externalUrl":null,"permalink":"/posts/gitops-external-secrets-vault/","section":"Posts","summary":"This is Part 4 of the Hashicorp Vault series. In Part 1, we bootstrapped Vault. In Part 2, we built our CI/CD pipeline. In Part 3, we finally killed the static kubeconfig.\n","title":"Vault (Part 4): Dynamic Secrets for Kubernetes Pods with External Secrets Operator","type":"posts"},{"content":"","date":"23 November 2025","externalUrl":null,"permalink":"/tags/devops/automation/ansible/","section":"Tags","summary":"","title":"devops/automation/ansible","type":"tags"},{"content":"","date":"23 November 2025","externalUrl":null,"permalink":"/tags/devops/automation/terraform/","section":"Tags","summary":"","title":"devops/automation/terraform","type":"tags"},{"content":"This is Part 3 of the Hashicorp Vault series. In Part 1, we bootstrapped Vault. In Part 2, we built the full CI/CD pipeline. Now, we\u0026rsquo;re solving the final, massive headache: getting secure, dynamic kubectl access.\nThe Problem: Static Kubeconfig Files # So, how do you talk to your cluster? kubectl, right? And how does kubectl know where to go? The good old kubeconfig file.\nLet\u0026rsquo;s be honest, we\u0026rsquo;ve all done the same thing: you scp the admin config from your control plane, save it to ~/.kube/config, and call it a day. I was no different.\ndev-rke2 (argo-cd) in ~ ❯ scp -i signed.pub -i ~/.ssh/id_ed25519 ubuntu@10.69.1.111:~/.kube/config ~/.kube/dev.yml config 100% 2971 370.8KB/s 00:00 dev-rke2 (argo-cd) in ~ ❯ cat ~/.kube/dev.yml apiVersion: v1 clusters: - cluster: certificate-authority-data: \u0026lt;\u0026lt;REDACTED_CERT_DATA\u0026gt;\u0026gt; server: https://10.69.1.111:6443 name: default contexts: - context: cluster: default user: default name: default current-context: default kind: Config preferences: {} users: - name: default user: client-certificate-data: \u0026lt;\u0026lt;REDACTED_CERT_DATA\u0026gt;\u0026gt; client-key-data: \u0026lt;\u0026lt;REDACTED_KEY_DATA\u0026gt;\u0026gt; But that file is a ticking time bomb. It\u0026rsquo;s not just a config; it\u0026rsquo;s a static, super-powered master key to your entire cluster. That client-certificate-data probably has cluster-admin rights and never expires.\nWhat happens when you lose your laptop? Or when someone leaves the team? You can\u0026rsquo;t just revoke one person\u0026rsquo;s \u0026ldquo;key.\u0026rdquo; You have to rotate certs for the entire cluster (and let\u0026rsquo;s be real, nobody does that). When everyone uses the same admin config, you have zero audit trail. You have no idea who deleted that deployment at 3 AM.\nThis isn\u0026rsquo;t a theoretical problem. This is exactly what happened to me. As I mentioned in Part 1, when my laptop died, I didn\u0026rsquo;t just lose my SSH keys; I lost my only kubeconfig file. I was completely, 100% locked out. I had to re-bootstrap the entire cluster from scratch. That was the moment I decided: no more static kubeconfigs. Ever.\nAfter I got the GitHub Actions OIDC flow working with Vault, I was motivated to do the same for kubectl. The goal: kubectl should authenticate me, as a human, and give me a temporary, role-based ticket.\nMy GitHub runner auth was my guide:\nRunner asks GitHub (OIDC Provider) for a token.\nRunner gives that token to Vault.\nVault verifies the token and grants access.\nSo, for kubectl, I just need to swap the pieces:\nThe \u0026ldquo;runner\u0026rdquo; is now my local machine.\nThe \u0026ldquo;target\u0026rdquo; is the Kubernetes API server.\n\u0026hellip;and I still need an OIDC provider.\nMy first thought was to use Keycloak, Authentik, or Authelia. But this immediately created a new chicken-and-egg problem: these tools run on Kubernetes. How do you deploy the auth app when you can\u0026rsquo;t even authenticate to the cluster to deploy anything? I could try to jam it in with Ansible, but debugging that without kubectl sounded like a special kind of hell.\nThen it hit me. Coincidentally, the tool I just set up to manage my secrets in the previous part also happened to be a OIDC provider.\nThe Solution: Vault as an OIDC Provider # However, before we can log in to Vault and get the JWT token just like the runner did with Github, there\u0026rsquo;s a catch. kubectl on its own is\u0026hellip; well, kinda dumb. It doesn\u0026rsquo;t know how to pop open a browser and say \u0026ldquo;log me in.\u0026rdquo; It just knows how to use a token if it has one.\nTo fix this, we need a helper: kubelogin. This is a small kubectl plugin that does the magic. When you run kubectl get nodes, kubelogin hijacks the process. It sees you don\u0026rsquo;t have a valid token, pops open your browser, and makes you log into Vault (with a username/password, just like any other website). Once you\u0026rsquo;re in, Vault generates a short-lived ID token and sends it back to kubelogin.\nBut here\u0026rsquo;s the really cool part. Vault doesn\u0026rsquo;t just give you a generic token. It stuffs it with claims - information about you.\nsub (subject): Your username (e.g., phuchoang)\ngroups: A list of groups you\u0026rsquo;re in (e.g., cluster-admins, dev-viewers)\niss (issuer) \u0026amp; aud (audience): Cryptographic proof saying, \u0026ldquo;This token definitely came from my Vault server and was meant for my Kubernetes cluster.\u0026rdquo;\nFinally, kubectl takes this shiny new token and shows it to the Kubernetes API server. The API server (which we\u0026rsquo;ll configure to trust Vault) checks the iss and aud fields. If they look good, it passes your groups claim to the RBAC (Role-Based Access Control) system. RBAC makes the final call. If you\u0026rsquo;re in the dev-viewers group and you try to kubectl create deployment, RBAC will shut you down. No-fly zone.\nHere\u0026rsquo;s the whole flow:\nThe kubelogin Side (Steps 1-6):\nYou type kubectl get nodes.\nkubectl/kubelogin sees you have no token (or it\u0026rsquo;s expired).\nA browser pops open. You log into Vault.\nVault sends back an ID token stuffed with your user and group info.\nkubelogin snags that token and caches it.\nkubectl (now happy) uses this token in its request.\nThe K8s API Server Side (Steps 7-11):\nkubectl sends the API request with the token.\nThe API server verifies the token\u0026rsquo;s signature against Vault\u0026rsquo;s.\nIf valid, it extracts your user and groups from the token.\nIt asks RBAC: \u0026ldquo;Hey, is a user from the cluster-admins group allowed to get nodes?\u0026rdquo;\nRBAC says \u0026ldquo;yep,\u0026rdquo; and you get your list of nodes.\nWith that background, let\u0026rsquo;s (again) jump into the implementation.\nStep 1: Configure Vault as an OIDC Provider # In the last part, getting the GitHub Actions runner to authenticate was way easier. The runner and the OIDC provider were both just \u0026ldquo;GitHub,\u0026rdquo; so they already trusted each other.\nNow, we\u0026rsquo;re connecting two totally different worlds: our local kubectl and our self-hosted Vault. This means we have to do a lot more manual wiring. And let\u0026rsquo;s be real, trying to click through 15 different settings in the clunky Vault UI is a recipe for error and a nightmare to reproduce.\nSo, Terraform to the rescue.\nWe\u0026rsquo;ll add this new configuration to our existing terraform-admin project. It\u0026rsquo;s a one-time setup, just like the JWT auth backend and SSH CA we did before. We\u0026rsquo;re basically going to teach Vault how to be a full-blown OIDC provider.\nThis breaks down into a few key parts:\nA user backend: How do we prove who we are? We\u0026rsquo;ll set up a simple userpass (username/password) backend for now. (Keep in mind, you could get fancy and use other auth methods like LDAP or even cloud auth).\nIdentity Groups: We\u0026rsquo;ll create our \u0026ldquo;teams\u0026rdquo; - admins, developers, and viewers.\nOIDC Scopes: We\u0026rsquo;ll define what info (like our group membership) to stuff inside the token.\nThe OIDC Provider: This is the actual endpoint that kubelogin will talk to, tying everything together.\nCreating Vault Identity Groups # First, we need to create our \u0026ldquo;teams\u0026rdquo; inside Vault. These are Identity Groups that will map directly to the RBAC (Role-Based Access Control) roles we\u0026rsquo;ll create in Kubernetes later. This is the crucial link between a Vault user and a Kubernetes permission.\nFile: terraform-admin/modules/vault-oidc-kubernetes/main.tf\n# Create Vault groups for Kubernetes RBAC resource \u0026#34;vault_identity_group\u0026#34; \u0026#34;kubernetes_admins\u0026#34; { name = \u0026#34;${var.env}-kubernetes-admins\u0026#34; type = \u0026#34;internal\u0026#34; policies = [] metadata = { environment = var.env purpose = \u0026#34;Kubernetes cluster administrators\u0026#34; } } resource \u0026#34;vault_identity_group\u0026#34; \u0026#34;kubernetes_developers\u0026#34; { name = \u0026#34;${var.env}-kubernetes-developers\u0026#34; type = \u0026#34;internal\u0026#34; policies = [] metadata = { environment = var.env purpose = \u0026#34;Kubernetes application developers\u0026#34; } } resource \u0026#34;vault_identity_group\u0026#34; \u0026#34;kubernetes_viewers\u0026#34; { name = \u0026#34;${var.env}-kubernetes-viewers\u0026#34; type = \u0026#34;internal\u0026#34; policies = [] metadata = { environment = var.env purpose = \u0026#34;Kubernetes read-only viewers\u0026#34; } } I\u0026rsquo;m setting up three groups to match a standard RBAC setup:\nkubernetes-admins: Full cluster-admin. The \u0026ldquo;god mode\u0026rdquo; users.\nkubernetes-developers: Can manage apps (pods, deployments, secrets), but can\u0026rsquo;t touch nodes or cluster settings.\nkubernetes-viewers: Look, don\u0026rsquo;t touch. Read-only access for everyone else.\nA quick heads-up: When Kubernetes receives these group names from Vault\u0026rsquo;s token, it will automatically add an oidc: prefix to them (e.g., oidc:dev-kubernetes-admins). This is a built-in K8s feature to prevent name collisions with its own internal groups. We\u0026rsquo;ll just need to remember this when we write our RBAC policies.\nConfiguring OIDC Scopes # Next, we need to tell Vault what data to put inside the OIDC token it generates. By default, a token is pretty empty. We use OIDC Scopes to \u0026ldquo;stuff\u0026rdquo; it with the info we need.\n# Create OIDC key for signing tokens resource \u0026#34;vault_identity_oidc_key\u0026#34; \u0026#34;kubernetes\u0026#34; { name = \u0026#34;${var.env}-kubernetes\u0026#34; allowed_client_ids = [\u0026#34;*\u0026#34;] rotation_period = 3600 verification_ttl = 3600 } # Create custom scope for email resource \u0026#34;vault_identity_oidc_scope\u0026#34; \u0026#34;email\u0026#34; { name = \u0026#34;${var.env}-email\u0026#34; description = \u0026#34;Email scope for ${var.env} Kubernetes\u0026#34; template = \u0026lt;\u0026lt;-EOT { \u0026#34;email\u0026#34;: {{identity.entity.metadata.email}} } EOT } # Create custom scope for profile resource \u0026#34;vault_identity_oidc_scope\u0026#34; \u0026#34;profile\u0026#34; { name = \u0026#34;${var.env}-profile\u0026#34; description = \u0026#34;Profile scope for ${var.env} Kubernetes\u0026#34; template = \u0026lt;\u0026lt;-EOT { \u0026#34;name\u0026#34;: {{identity.entity.name}}, \u0026#34;preferred_username\u0026#34;: {{identity.entity.aliases.${var.userpass_auth_accessor}.name}} } EOT } # Create custom scope for groups resource \u0026#34;vault_identity_oidc_scope\u0026#34; \u0026#34;groups\u0026#34; { name = \u0026#34;${var.env}-groups\u0026#34; description = \u0026#34;Groups scope for ${var.env} Kubernetes\u0026#34; template = \u0026lt;\u0026lt;-EOT { \u0026#34;groups\u0026#34;: {{identity.entity.groups.names}} } EOT } We need three custom scopes:\nemail: Puts the user\u0026rsquo;s email in the token. This is great for audit logs.\nprofile: Adds the user\u0026rsquo;s name. This is what will show up as the \u0026ldquo;user\u0026rdquo; in kubectl audit logs.\ngroups: This is the most important one. It adds a list of all the Vault groups the user is in (like dev-kubernetes-admins). This is the claim that Kubernetes RBAC will use to decide what you\u0026rsquo;re-allowed to do.\nYou\u0026rsquo;ll also see a vault_identity_oidc_key resource. This is just Vault creating the cryptographic key it will use to sign all the OIDC tokens it issues.\nThe magic is in the template for each scope. We\u0026rsquo;re using Vault\u0026rsquo;s templating to tell it exactly where to pull this data from the user\u0026rsquo;s \u0026ldquo;Identity Entity\u0026rdquo; when they log in.\nA quick, lazy confession: The preferred_username I\u0026rsquo;m creating in the profile scope is basically the sub (subject) claim. I probably should have used the standard sub field, but I was too lazy to go back and change everything. It works. 😅\nCreating the OIDC Provider and Client # Okay, we have our groups and we know what our token will look like. Now we need to create the actual OIDC Provider - this is the endpoint that kubelogin will hit.\n# Create OIDC provider resource \u0026#34;vault_identity_oidc_provider\u0026#34; \u0026#34;kubernetes\u0026#34; { name = var.env https_enabled = true issuer_host = replace(var.vault_addr, \u0026#34;https://\u0026#34;, \u0026#34;\u0026#34;) allowed_client_ids = [ vault_identity_oidc_client.kubernetes.client_id ] scopes_supported = [ vault_identity_oidc_scope.email.name, vault_identity_oidc_scope.profile.name, vault_identity_oidc_scope.groups.name, ] } # Create OIDC assignment (controls who can authenticate) resource \u0026#34;vault_identity_oidc_assignment\u0026#34; \u0026#34;kubernetes\u0026#34; { name = \u0026#34;${var.env}-kubernetes-users\u0026#34; group_ids = [ vault_identity_group.kubernetes_admins.id, vault_identity_group.kubernetes_developers.id, vault_identity_group.kubernetes_viewers.id, ] } # Create OIDC client for Kubernetes resource \u0026#34;vault_identity_oidc_client\u0026#34; \u0026#34;kubernetes\u0026#34; { name = \u0026#34;${var.env}-kubernetes\u0026#34; key = vault_identity_oidc_key.kubernetes.name redirect_uris = var.redirect_uris assignments = [ vault_identity_oidc_assignment.kubernetes.name ] id_token_ttl = 3600 # 1 hour access_token_ttl = 3600 # 1 hour client_type = \u0026#34;public\u0026#34; } # Store OIDC client_id in Vault KV for use by Ansible resource \u0026#34;vault_generic_secret\u0026#34; \u0026#34;oidc_client_id\u0026#34; { path = \u0026#34;kv/${var.env}/oidc\u0026#34; data_json = jsonencode({ client_id = vault_identity_oidc_client.kubernetes.client_id }) } This piece of configuration ties everything together:\nIt defines the OIDC client (our \u0026ldquo;application,\u0026rdquo; which in this case is kubelogin).\nIt sets the redirect_uris. This is a critical security step. It\u0026rsquo;s an allow-list of URLs that Vault is allowed to send a user back to after they log in. Since kubelogin runs a tiny web server on your local machine to catch the token, we must add http://localhost:8000 (and 18000 as a backup) here.\nIt sets the token TTL. I set it to 3600 seconds (1 hour). This feels like a good balance between security (short-lived tokens) and usability (not having to log in every 5 minutes).\nIt assigns our admins, developers, and viewers groups to this provider, so only members of these groups can log in.\nA few key things to notice in the code:\nclient_type = \u0026quot;public\u0026quot;: This is important. It means our client (kubelogin) doesn\u0026rsquo;t need a \u0026ldquo;client secret\u0026rdquo; to authenticate. This is the correct choice for a CLI tool running on a user\u0026rsquo;s workstation where a secret can\u0026rsquo;t be securely stored.\nvault_generic_secret \u0026quot;oidc_client_id\u0026quot;: This is the most important handoff in this whole file. Vault will generate a random, unique client_id for our app. We can\u0026rsquo;t know this ID in advance. So, we use this Terraform resource to read that generated ID and write it back into a Vault KV secret at kv/dev/oidc.\nWhy? Because our Ansible script (in the next step) will need to read this exact ID to configure the Kubernetes API server. This is how we pass this critical, dynamic value from our terraform-admin setup to our cluster bootstrap process. Module Usage in Main Configuration # Finally, we just need to call this new vault-oidc-kubernetes module from our main terraform-admin/main.tf file. We\u0026rsquo;ll call it twice: once for dev and once for prod.\nFile: terraform-admin/main.tf\n# Vault OIDC Provider for Kubernetes (Dev) module \u0026#34;vault_oidc_dev\u0026#34; { source = \u0026#34;./modules/vault-oidc-kubernetes\u0026#34; env = \u0026#34;dev\u0026#34; vault_addr = var.vault_addr userpass_auth_accessor = vault_auth_backend.userpass.accessor redirect_uris = [ \u0026#34;http://localhost:8000\u0026#34;, # kubelogin default \u0026#34;http://localhost:18000\u0026#34;, # kubelogin alternative ] } # Vault OIDC Provider for Kubernetes (Prod) module \u0026#34;vault_oidc_prod\u0026#34; { source = \u0026#34;./modules/vault-oidc-kubernetes\u0026#34; env = \u0026#34;prod\u0026#34; vault_addr = var.vault_addr userpass_auth_accessor = vault_auth_backend.userpass.accessor redirect_uris = [ \u0026#34;http://localhost:8000\u0026#34;, # kubelogin default \u0026#34;http://localhost:18000\u0026#34;, # kubelogin alternative ] } This creates two completely separate and isolated OIDC providers. A user in the dev-kubernetes-admins group will have zero ability to even try to log in to the prod environment, which is exactly what we want.\nYou\u0026rsquo;ll also notice we pass in that weird userpass_auth_accessor variable. This is a bit of Vault-Terraform magic. It\u0026rsquo;s the unique ID of the userpass auth method we set up.\nWhy? Go back and look at the profile scope template: ...{{identity.entity.aliases.${var.userpass_auth_accessor}.name}}\nThis is how the template finds the user\u0026rsquo;s alias (their username) from the specific auth method they used to log in. It\u0026rsquo;s a bit of a mouthful, but it\u0026rsquo;s the glue that connects the user\u0026rsquo;s login action to their Vault identity.\nStep 2: User Management with Terraform # Okay, so we\u0026rsquo;ve built the \u0026ldquo;house\u0026rdquo; (the OIDC provider, auth methods, and empty groups). Now we need to create the \u0026ldquo;guest list\u0026rdquo; - the actual user accounts and the keys to let them in.\nThis is the whole point of our terraform-admin project: automating the \u0026ldquo;day 2\u0026rdquo; operations, not just the \u0026ldquo;day 1\u0026rdquo; setup. We\u0026rsquo;re not going to click \u0026ldquo;add user\u0026rdquo; in the UI like savages.\nMy User Management Strategy # After some trial and error, I landed on a pattern that is clean, reusable, and, most importantly, easy to manage from a single file.\nDefine All Users in terraform.tfvars: I want a single, human-readable file to define who gets access to what.\nUse a Simple \u0026ldquo;User Factory\u0026rdquo; Module: This module\u0026rsquo;s only job is to create a single user\u0026rsquo;s login, their \u0026ldquo;entity\u0026rdquo; (their identity), and the \u0026ldquo;alias\u0026rdquo; that links them.\nHandle Group Memberships Separately: This is the most important part. We\u0026rsquo;ll create all the users first, then add them to their groups. This avoids a world of pain with Terraform\u0026rsquo;s dependency cycles.\n1. Defining Users with Variables (The \u0026ldquo;Guest List\u0026rdquo;) # First, let\u0026rsquo;s define what a \u0026ldquo;user\u0026rdquo; looks like in our terraform-admin/variables.tf. I wanted a map where I could see, at a glance, every user and their exact permissions for each environment.\nFile: terraform-admin/variables.tf\nvariable \u0026#34;users\u0026#34; { description = \u0026#34;Map of users to create with their group memberships\u0026#34; type = map(object({ email = string password = string groups = object({ dev_role = optional(string) # \u0026#34;admins\u0026#34;, \u0026#34;developers\u0026#34;, \u0026#34;viewers\u0026#34;, or null prod_role = optional(string) # \u0026#34;admins\u0026#34;, \u0026#34;developers\u0026#34;, \u0026#34;viewers\u0026#34;, or null }) })) default = {} validation { condition = alltrue([ for user in var.users : alltrue([ for role in [user.groups.dev_role, user.groups.prod_role] : role == null || contains([\u0026#34;admins\u0026#34;, \u0026#34;developers\u0026#34;, \u0026#34;viewers\u0026#34;], role) ]) ]) error_message = \u0026#34;Role must be one of: \u0026#39;admins\u0026#39;, \u0026#39;developers\u0026#39;, \u0026#39;viewers\u0026#39;, or null.\u0026#34; } } This structure is super flexible. The groups object lets me give a user dev access, prod access, both, or neither.\nHere’s what it looks like in practice. This is the only file I\u0026rsquo;ll need to edit when a new person joins the team.\nFile: terraform-admin/terraform.tfvars.example\nusers = { # Developer with dev access only \u0026#34;developer\u0026#34; = { email = \u0026#34;developer@example.com\u0026#34; password = \u0026#34;CHANGE-ME-SECURE-PASSWORD\u0026#34; groups = { dev_role = \u0026#34;developers\u0026#34; prod_role = null # No prod access } } # Viewer with read-only access to both environments \u0026#34;viewer\u0026#34; = { email = \u0026#34;viewer@example.com\u0026#34; password = \u0026#34;CHANGE-ME-SECURE-PASSWORD\u0026#34; groups = { dev_role = \u0026#34;viewers\u0026#34; prod_role = \u0026#34;viewers\u0026#34; } } # Prod-only admin \u0026#34;prod-admin\u0026#34; = { email = \u0026#34;prod.admin@example.com\u0026#34; password = \u0026#34;CHANGE-ME-SECURE-PASSWORD\u0026#34; groups = { dev_role = null # No dev access prod_role = \u0026#34;admins\u0026#34; } } } 2. The User Module (The \u0026ldquo;User Factory\u0026rdquo;) # Next, we need a simple, reusable module that knows how to create one user. This module does three distinct things to create a complete identity in Vault:\nvault_generic_endpoint \u0026quot;user\u0026quot;: This creates the userpass login (the actual username and password they\u0026rsquo;ll type in the browser).\nvault_identity_entity \u0026quot;user\u0026quot;: This creates the Entity. Think of this as the user\u0026rsquo;s \u0026ldquo;soul\u0026rdquo; or their core identity in Vault. It holds metadata like their email.\nvault_identity_entity_alias \u0026quot;user\u0026quot;: This is the glue. It links the userpass login (the \u0026ldquo;door\u0026rdquo;) to the entity (the \u0026ldquo;person\u0026rdquo;).\nFile: terraform-admin/modules/vault-user/main.tf\n# 1. Create the login (username/password) resource \u0026#34;vault_generic_endpoint\u0026#34; \u0026#34;user\u0026#34; { path = \u0026#34;auth/${var.userpass_mount_path}/users/${var.username}\u0026#34; ignore_absent_fields = true data_json = jsonencode({ password = var.password token_ttl = var.token_ttl token_max_ttl = var.token_max_ttl }) } # 2. Create the \u0026#34;person\u0026#34; (the Entity) resource \u0026#34;vault_identity_entity\u0026#34; \u0026#34;user\u0026#34; { name = var.username metadata = { email = var.email } } # 3. Link the login to the person (the Alias) resource \u0026#34;vault_identity_entity_alias\u0026#34; \u0026#34;user\u0026#34; { name = var.username mount_accessor = var.userpass_auth_accessor canonical_id = vault_identity_entity.user.id } Notice what\u0026rsquo;s missing? I\u0026rsquo;m not managing group memberships here. We pass in an empty group_ids = [] list. This is intentional. Trying to manage groups inside this for_each\u0026rsquo;d module while the groups themselves are also being created in a for_each loop is a one-way ticket to dependency-cycle hell. Trust me, don\u0026rsquo;t do it.\n3. The Main Configuration (The \u0026ldquo;Sorting Hat\u0026rdquo;) # This is the \u0026ldquo;master orchestrator\u0026rdquo; file (terraform-admin/users.tf) that brings it all together. It\u0026rsquo;s a bit of Terraform data-gymnastics, but it\u0026rsquo;s very clean once it\u0026rsquo;s set up.\nFile: terraform-admin/users.tf\nlocals { # Create a flat map of user-to-group assignments with static keys # This avoids Terraform\u0026#39;s \u0026#34;for_each with unknown values\u0026#34; limitation user_group_memberships = merge(flatten([ for username, user in var.users : [ user.groups.dev_role != null ? { \u0026#34;${username}-dev-${user.groups.dev_role}\u0026#34; = { username = username env = \u0026#34;dev\u0026#34; role = user.groups.dev_role } } : {}, user.groups.prod_role != null ? { \u0026#34;${username}-prod-${user.groups.prod_role}\u0026#34; = { username = username env = \u0026#34;prod\u0026#34; role = user.groups.prod_role } } : {} ] ])...) } # Create users (without group assignments) module \u0026#34;vault_users\u0026#34; { source = \u0026#34;./modules/vault-user\u0026#34; for_each = var.users username = each.key password = each.value.password email = each.value.email group_ids = [] # Groups will be managed separately userpass_mount_path = \u0026#34;userpass\u0026#34; userpass_auth_accessor = vault_auth_backend.userpass.accessor depends_on = [ module.vault_oidc_dev, module.vault_oidc_prod ] } # Manage group memberships with static keys # Each resource assigns one user to one group resource \u0026#34;vault_identity_group_member_entity_ids\u0026#34; \u0026#34;user_group_assignments\u0026#34; { for_each = local.user_group_memberships group_id = each.value.env == \u0026#34;dev\u0026#34; ? ( module.vault_oidc_dev.group_ids[each.value.role] ) : ( module.vault_oidc_prod.group_ids[each.value.role] ) member_entity_ids = [module.vault_users[each.value.username].entity_id] exclusive = false depends_on = [ module.vault_users, module.vault_oidc_dev, module.vault_oidc_prod ] } And that\u0026rsquo;s it! It looks complex, but the flow is simple:\nlocals transforms our human-friendly tfvars file into the group_memberships map that Terraform needs. module \u0026quot;vault_users\u0026quot; creates all the users in parallel (but assigns them no groups). vault_identity_group_member_entity_ids then acts like the \u0026ldquo;Sorting Hat,\u0026rdquo; looping through each group and adding all its members in one go. This two-step process (create users, then assign groups) neatly sidesteps all the complex dependency issues and makes the whole setup clean and manageable.\nStep 3: Configure the Kubernetes API Server for OIDC # Okay, so Vault is now ready to act as an OIDC provider. It\u0026rsquo;s sitting there, ready to issue tokens to our users.\nNow we have to do the other half of the handshake: we need to tell our Kubernetes cluster to trust it.\nIf Vault is the bouncer, we need to give the Kubernetes API server a \u0026ldquo;guest list\u0026rdquo; and tell it, \u0026ldquo;If someone shows up with an ID from this bouncer, let them in.\u0026rdquo;\nThis is the \u0026ldquo;what we\u0026rsquo;re doing\u0026rdquo; part. Since I\u0026rsquo;m using Ansible to provision RKE2, I\u0026rsquo;ll be editing my Ansible template for the RKE2 server config.\nUpdating RKE2 Server Configuration # This is the core of the connection. We need to add several kube-apiserver-arg flags to the RKE2 config. These are just command-line arguments that tell the Kubernetes API server how to behave.\nFile: ansible/roles/add_server/templates/rke2-server-config.j2\nwrite-kubeconfig-mode: \u0026#34;0644\u0026#34; token: {{ rke2_token }} tls-san: - {{ vip }} {% for host in groups[\u0026#39;servers\u0026#39;] %} - {{ hostvars[host][\u0026#39;ansible_host\u0026#39;] }} {% endfor %} node-label: - server=true disable-cloud-controller: true disable: - rke2-ingress-nginx kube-proxy-arg: - \u0026#34;proxy-mode=ipvs\u0026#34; - \u0026#34;ipvs-strict-arp=true\u0026#34; kube-proxy-extra-mount: - \u0026#34;/lib/modules:/lib/modules:ro\u0026#34; kube-apiserver-arg: - \u0026#34;oidc-issuer-url={{ oidc_issuer_url }}\u0026#34; - \u0026#34;oidc-client-id={{ oidc_client_id }}\u0026#34; - \u0026#34;oidc-username-claim=preferred_username\u0026#34; - \u0026#34;oidc-groups-claim=groups\u0026#34; - \u0026#34;oidc-groups-prefix=oidc:\u0026#34; Here\u0026rsquo;s a breakdown of the exact flags we need to add:\noidc-issuer-url: This is the most important one. It\u0026rsquo;s the URL to our Vault OIDC provider. When the API server gets a token, it will use this URL to go fetch Vault\u0026rsquo;s public keys (from a well-known endpoint) and verify the token\u0026rsquo;s signature. This is how it knows the token is legit.\noidc-client-id: This is the \u0026ldquo;audience\u0026rdquo; check. It tells the API server, \u0026ldquo;Only accept tokens that were meant for me.\u0026rdquo; Remember that client_id Vault generated for us? This is it. If a token doesn\u0026rsquo;t have this exact ID in its aud claim, the API server will reject it.\noidc-username-claim: \u0026ldquo;When you get a valid token, which field has the user\u0026rsquo;s name?\u0026rdquo; We set this up in Step 1 to be preferred_username.\noidc-groups-claim: \u0026ldquo;And which field has their list of groups?\u0026rdquo; We called ours groups.\noidc-groups-prefix: This is a great best practice. It automatically sticks oidc: onto the front of any group name it gets from the token (e.g., dev-kubernetes-admins becomes oidc:dev-kubernetes-admins). This makes it super obvious in our RBAC policies which groups are from OIDC versus built-in K8s groups.\nConnecting the Dots: From Vault to GitHub to Ansible # This is where all the pieces we\u0026rsquo;ve been building finally click together. How does our Ansible script, running in a GitHub Actions runner, get these values?\n1. Ansible Needs the Variables: First, I tell Ansible to expect these values from environment variables. My group_vars/all/main.yaml file looks for them.\noidc_issuer_url: \u0026#34;{{ lookup(\u0026#39;env\u0026#39;, \u0026#39;OIDC_ISSUER_URL\u0026#39;) }}\u0026#34; oidc_client_id: \u0026#34;{{ lookup(\u0026#39;env\u0026#39;, \u0026#39;OIDC_CLIENT_ID\u0026#39;) }}\u0026#34; 2. GitHub Actions Provides the Variables: This is the beautiful part. Our GitHub Actions workflow is responsible for fetching these values and making them available to Ansible.\nOIDC_CLIENT_ID: This is the critical handoff from Step 1. We can\u0026rsquo;t hardcode this value, because Vault generated it for us. Our terraform-admin run saved that dynamic ID into a KV secret (kv/dev/data/oidc). Now, our GitHub workflow just reads that value right back out of Vault. OIDC_ISSUER_URL: This one is easy. We can just construct it. It\u0026rsquo;s just our VAULT_ADDR variable plus the static path /v1/identity/oidc/provider/ and the environment name (dev or prod). Here\u0026rsquo;s the snippet from the GitHub Actions workflow that does this work. It logs into Vault, grabs the dynamic client_id from the K/V store, and builds the issuer_url from variables.\n- name: Import Secrets from Vault uses: hashicorp/vault-action@v3 id: vault_auth with: method: jwt url: ${{ vars.VAULT_ADDR }} role: ${{ steps.set-role.outputs.role }} outputToken: true secrets: | kv/shared/data/cloudflare * | SSL_ ; kv/${{vars.ENV_NAME}}/data/ip * | IP_ ; kv/${{vars.ENV_NAME}}/data/rke2 * | RKE2_ ; kv/${{vars.ENV_NAME}}/data/oidc client_id | OIDC_CLIENT_ID ; - name: Set OIDC Issuer URL run: | echo \u0026#34;OIDC_ISSUER_URL=${{ vars.VAULT_ADDR }}/v1/identity/oidc/provider/${{ vars.ENV_NAME }}\u0026#34; \u0026gt;\u0026gt; $GITHUB_ENV Once these are exported as environment variables, Ansible picks them up, templates them into the rke2-server-config.j2 file, and restarts the RKE2 server.\nAnd just like that, our Kubernetes cluster now fully trusts our Vault server as an identity provider.\nStep 4: Deploy Kubernetes RBAC Policies # We\u0026rsquo;re on the home stretch.\nSo far:\nVault is our \u0026ldquo;bouncer,\u0026rdquo; ready to issue signed ID cards (OIDC tokens).\nThe Kubernetes API server is our \u0026ldquo;club,\u0026rdquo; and we\u0026rsquo;ve configured it to trust that bouncer (via the OIDC flags).\nNow for the final and most important step: writing the guest list.\nWe need to tell Kubernetes, \u0026ldquo;Okay, when someone does show up with a valid ID card that says they\u0026rsquo;re in the oidc:dev-kubernetes-admins group, what are they actually allowed to do?\u0026rdquo;\nThis is all handled by standard Kubernetes RBAC (Role-Based Access Control) manifests. We\u0026rsquo;ll create ClusterRoleBindings that link our OIDC groups to specific permissions.\n1. The \u0026lsquo;Admins\u0026rsquo; (God Mode) # This one is the easiest. We need to give our admins group full, unrestricted, \u0026ldquo;do-anything\u0026rdquo; power. In Kubernetes, this is the built-in cluster-admin role.\nFile: ansible/roles/add_server/templates/oidc-rbac-cluster-admin.yaml.j2\n--- # Grant cluster-admin to kubernetes-admins group apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: oidc-cluster-admins annotations: description: \u0026#34;Grants cluster-admin access to kubernetes-admins group\u0026#34; roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: cluster-admin subjects: - kind: Group name: oidc:{{ env }}-kubernetes-admins apiGroup: rbac.authorization.k8s.io We just need to create a ClusterRoleBinding to \u0026ldquo;bind\u0026rdquo; our Vault group to this all-powerful role.\nThe single most important line in this file is the name under subjects. It must be oidc:{{ env }}-kubernetes-admins.\nThat oidc: prefix isn\u0026rsquo;t optional. It\u0026rsquo;s the prefix we told the API server to add (--oidc-groups-prefix=oidc:). If you forget this, the binding won\u0026rsquo;t match, and your \u0026ldquo;admin\u0026rdquo; user will log in with zero permissions, which is a very confusing 20 minutes of debugging.\n2. The \u0026lsquo;Developers\u0026rsquo; (The \u0026ldquo;Do-ers\u0026rdquo;) # This is the most interesting role and the one that required the most thought. I don\u0026rsquo;t want my developers to have cluster-admin - that\u0026rsquo;s how you get \u0026ldquo;oops, I deleted the prod namespace\u0026rdquo; accidents. But I also despise being a bottleneck. If a developer needs to check logs or deploy a new image, they shouldn\u0026rsquo;t have to file a ticket with me.\nThe philosophy for this developer ClusterRole is: Freedom to ship, not freedom to break.\nFile: ansible/roles/add_server/templates/oidc-rbac-developer.yaml.j2\n--- # Developer role with read/write access to common resources apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: developer rules: # Core resources - apiGroups: [\u0026#34;\u0026#34;] resources: [ \u0026#34;pods\u0026#34;, \u0026#34;pods/log\u0026#34;, \u0026#34;pods/exec\u0026#34;, \u0026#34;services\u0026#34;, \u0026#34;configmaps\u0026#34;, \u0026#34;secrets\u0026#34;, \u0026#34;persistentvolumeclaims\u0026#34;, ] verbs: [\u0026#34;get\u0026#34;, \u0026#34;list\u0026#34;, \u0026#34;watch\u0026#34;, \u0026#34;create\u0026#34;, \u0026#34;update\u0026#34;, \u0026#34;patch\u0026#34;, \u0026#34;delete\u0026#34;] # Apps - apiGroups: [\u0026#34;apps\u0026#34;] resources: [\u0026#34;deployments\u0026#34;, \u0026#34;statefulsets\u0026#34;, \u0026#34;daemonsets\u0026#34;, \u0026#34;replicasets\u0026#34;] verbs: [\u0026#34;get\u0026#34;, \u0026#34;list\u0026#34;, \u0026#34;watch\u0026#34;, \u0026#34;create\u0026#34;, \u0026#34;update\u0026#34;, \u0026#34;patch\u0026#34;, \u0026#34;delete\u0026#34;] # Batch jobs - apiGroups: [\u0026#34;batch\u0026#34;] resources: [\u0026#34;jobs\u0026#34;, \u0026#34;cronjobs\u0026#34;] verbs: [\u0026#34;get\u0026#34;, \u0026#34;list\u0026#34;, \u0026#34;watch\u0026#34;, \u0026#34;create\u0026#34;, \u0026#34;update\u0026#34;, \u0026#34;patch\u0026#34;, \u0026#34;delete\u0026#34;] # Networking - apiGroups: [\u0026#34;networking.k8s.io\u0026#34;] resources: [\u0026#34;ingresses\u0026#34;, \u0026#34;networkpolicies\u0026#34;] verbs: [\u0026#34;get\u0026#34;, \u0026#34;list\u0026#34;, \u0026#34;watch\u0026#34;, \u0026#34;create\u0026#34;, \u0026#34;update\u0026#34;, \u0026#34;patch\u0026#34;, \u0026#34;delete\u0026#34;] # Autoscaling - apiGroups: [\u0026#34;autoscaling\u0026#34;] resources: [\u0026#34;horizontalpodautoscalers\u0026#34;] verbs: [\u0026#34;get\u0026#34;, \u0026#34;list\u0026#34;, \u0026#34;watch\u0026#34;, \u0026#34;create\u0026#34;, \u0026#34;update\u0026#34;, \u0026#34;patch\u0026#34;, \u0026#34;delete\u0026#34;] # Read-only access to nodes and namespaces - apiGroups: [\u0026#34;\u0026#34;] resources: [\u0026#34;nodes\u0026#34;, \u0026#34;namespaces\u0026#34;] verbs: [\u0026#34;get\u0026#34;, \u0026#34;list\u0026#34;, \u0026#34;watch\u0026#34;] --- # Bind developer role to kubernetes-developers group apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: oidc-developers annotations: description: \u0026#34;Grants developer access to kubernetes-developers group\u0026#34; roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: developer subjects: - kind: Group name: oidc:{{ env }}-kubernetes-developers apiGroup: rbac.authorization.k8s.io So, what can they do?\nThey CAN manage the full lifecycle of applications. This means create, update, delete, and get for things like pods, deployments, services, configmaps, secrets, ingresses, and jobs.\nThey CAN debug. This is crucial. They have pods/exec (to get into a container) and pods/log (to read logs).\nThey CAN\u0026rsquo;T touch the cluster infrastructure. They can\u0026rsquo;t delete nodes. They can\u0026rsquo;t create or delete namespaces (though they can view them). And they absolutely cannot modify ClusterRoles or ClusterRoleBindings (especially their own!).\nThis strikes the perfect balance, in my opinion. They can deploy and manage their apps all day long without ever having the power to take down the whole cluster.\n3. The \u0026lsquo;Viewers\u0026rsquo; (The Easy Win) # This one is another simple one. I don\u0026rsquo;t need to create a fancy custom role because Kubernetes already gives us a built-in ClusterRole called view.\nFile: ansible/roles/add_server/templates/oidc-rbac-viewer.yaml.j2\n--- # Grant view-only access to kubernetes-viewers group apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: name: oidc-viewers annotations: description: \u0026#34;Grants read-only access to kubernetes-viewers group\u0026#34; roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole name: view subjects: - kind: Group name: oidc:{{ env }}-kubernetes-viewers apiGroup: rbac.authorization.k8s.io It does exactly what it says on the tin: get, list, and watch permissions on almost everything, but zero write access and (importantly) no permission to read secrets.\nWe just create one more ClusterRoleBinding to map our oidc:{{ env }}-kubernetes-viewers group directly to this view role. This is perfect for new team members, auditors, or anyone in the \u0026ldquo;just-looking\u0026rdquo; category.\nLocal Authentication with kubelogin (The Magic) # So, we have Vault ready to give out tokens, and K8s ready to accept them. How do we glue our local kubectl command to this new flow?\nAs I mentioned, kubectl is dumb. It only knows how to use a token, not how to get one from a browser. We need a translator. That translator is a slick little kubectl plugin called kubelogin.\nIt\u0026rsquo;s a small helper that sits between kubectl and your OIDC provider, and its only job is to handle the browser-based login flow.\nint128/kubelogin kubectl plugin for Kubernetes OpenID Connect authentication (kubectl oidc-login) Go 2375 245 Installing kubelogin # # Install kubelogin brew install int128/kubelogin/kubelogin # macOS # or kubectl krew install oidc-login # via krew Configuring Your New kubeconfig # This is the last kubeconfig file you\u0026rsquo;ll ever need to build by hand. Instead of having that massive, static client-certificate-data block, our new users section looks totally different. We\u0026rsquo;re going to use an exec block.\napiVersion: v1 kind: Config clusters: - cluster: server: https://10.69.1.110:6443 certificate-authority-data: \u0026lt;CA_DATA\u0026gt; name: dev contexts: - context: cluster: dev user: oidc name: dev current-context: dev users: - name: oidc user: exec: apiVersion: client.authentication.k8s.io/v1beta1 command: kubectl args: - oidc-login - get-token - --oidc-issuer-url=https://vault.example.com/v1/identity/oidc/provider/dev - --oidc-client-id=\u0026lt;CLIENT_ID\u0026gt; - --oidc-extra-scope=email - --oidc-extra-scope=profile - --oidc-extra-scope=groups This exec block basically tells kubectl:\n\u0026ldquo;Hey, before you try to talk to the cluster, I want you to execute a command first. Run kubectl oidc-login get-token and pass it all these arguments. That command will spit out a token, and that\u0026rsquo;s what you\u0026rsquo;ll use for authentication.\u0026rdquo;\nThe arguments we give it are exactly what we just configured in Vault and K8s:\n--oidc-issuer-url: The URL to our Vault OIDC provider (e.g., https://vault.example.com/.../provider/dev).\n--oidc-client-id: The dynamic, unique ID that terraform-admin generated and stored in our KV store.\n--oidc-extra-scope: We explicitly ask for the email, profile, and (most importantly) groups scopes that we created.\nRunning kubectl # This is the payoff for all that work. You open your terminal for the first time and execute kubectl get nodes. Here\u0026rsquo;s what happens:\nSummary \u0026amp; Next Steps # We\u0026rsquo;ve finally done it. We\u0026rsquo;ve killed the static kubeconfig.\nEvery single API request from kubectl is now authenticated with a short-lived, auditable JWT token. Our user identities are centralized in Vault. We can grant (and revoke!) access by simply adding or removing a user from a Vault group in Terraform. And because of the groups and email claims, we have a clear audit trail showing exactly who did what.\nThis completes the human access part of the story.\nIn the next and final part of this series, we\u0026rsquo;ll tackle the last piece: pod access. We\u0026rsquo;ll integrate the External Secrets Operator with Vault, allowing our Kubernetes pods to fetch secrets directly from Vault using their own service account tokens.\n","date":"23 November 2025","externalUrl":null,"permalink":"/posts/gitops-kubernetes-oidc-vault/","section":"Posts","summary":"This is Part 3 of the Hashicorp Vault series. In Part 1, we bootstrapped Vault. In Part 2, we built the full CI/CD pipeline. Now, we’re solving the final, massive headache: getting secure, dynamic kubectl access.\n","title":"Vault (Part 3): Secure Kubernetes Authentication with HashiCorp Vault OIDC","type":"posts"},{"content":"","date":"17 November 2025","externalUrl":null,"permalink":"/tags/network/self-hosted/tailscale/","section":"Tags","summary":"","title":"network/self-hosted/tailscale","type":"tags"},{"content":"This is Part 2 of the Hashicorp Vault series. Part 1 covered bootstrapping and configuring HashiCorp Vault (the one-time terraform-admin work) and the design goals for Zero Trust secrets management.\nWith Vault configured and the auth roles in place, it\u0026rsquo;s time to wire up the CI/CD pipeline. The workflows in .github/workflows will authenticate to Vault, fetch secrets, and then provision and configure the infrastructure.\nI have two main workflows:\nlint.yml: Runs cheap and fast quality checks (Terraform fmt, Ansible lint) on every PR. terraform-ansible.yml: The main pipeline that provisions and bootstraps the cluster. But first, there\u0026rsquo;s a problem to solve. GitHub-hosted runners are ephemeral and live on GitHub\u0026rsquo;s network. They can\u0026rsquo;t reach the private IPs (10.69.x.x) in my homelab. As mentioned earlier, I use Tailscale to solve this.\nSetting Up Tailscale for Secure Network Access # If you want to read more about my homelab network setup, you can read it here\nMy entire homelab is on a Tailscale network, a zero-config WireGuard-based mesh VPN. This setup allows a GitHub runner to join my network as an ephemeral node for the duration of a job and get direct access to my private IPs. When the job ends, the node disappears. You can even use Tailscale\u0026rsquo;s ACLs to limit the IP range this node can access.\nHere\u0026rsquo;s how to set it up:\nCreate a tag: Create at least one tag for your ephemeral nodes, for example, tag:ci. The access permissions you grant to this tag will apply to all nodes created by the workflow. Set up an OAuth client: Follow the guide to set up a Tailscale OAuth client. You will need the Client ID and Client secret. Create GitHub Secrets: Create two repository secrets, TS_OAUTH_CLIENT_ID and TS_OAUTH_SECRET, with the values from the previous step. Note: It\u0026rsquo;s worth mentioning that these are the only static, long-lived secrets I store in GitHub. Everything else is fetched dynamically from Vault.\nAdd to workflow: In your GitHub Actions workflow, use the official Tailscale action. - name: Tailscale uses: tailscale/github-action@v4 with: oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }} oauth-secret: ${{ secrets.TS_OAUTH_SECRET }} tags: tag:ci What this step will do:\nIt authenticates using short-lived OAuth credentials, not a long-lived auth key. tags: tag:ci labels the new ephemeral node so my network ACLs can be applied. This step must run before any job step that needs to talk to Vault or Proxmox. Setting Up Linting for Code Quality # Migrating to GitHub Actions also unlocked an easy win: automated linting. Linting is cheap, pays off immediately, and gives fast feedback on every pull request by catching syntax and formatting errors before they get merged.\nThis repo runs two parallel lint jobs: one for Terraform and one for Ansible.\nTerraform Lint Job (excerpt from .github/workflows/lint.yml):\nterraform-lint: name: Terraform Lint runs-on: ubuntu-latest defaults: run: working-directory: ./terraform-provision steps: - name: Checkout uses: actions/checkout@v4 - name: Setup Terraform uses: hashicorp/setup-terraform@v3 with: terraform_version: 1.6.6 - name: Terraform Init run: terraform init -backend=false - name: Terraform Format Check run: terraform fmt -check -recursive - name: Terraform Validate run: terraform validate Ansible Lint Job (excerpt):\nansible-lint: name: Ansible Lint runs-on: ubuntu-latest defaults: run: working-directory: ./ansible steps: - name: Checkout uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v5 with: python-version: \u0026#34;3.13\u0026#34; - name: Install ansible-lint run: pip install ansible-lint - name: Install Ansible collections run: ansible-galaxy install -r requirements.yml - name: Run ansible-lint run: ansible-lint Finally, I use a path-based trigger to keep these lint jobs focused. They only run when infrastructure, playbook, or workflow files change, which saves CI minutes.\non: pull_request: paths: - \u0026#34;terraform-admin/**\u0026#34; - \u0026#34;terraform-provision/**\u0026#34; - \u0026#34;ansible/**\u0026#34; - \u0026#34;.github/workflows/lint.yml\u0026#34; - \u0026#34;.github/workflows/terraform-ansible.yml\u0026#34; The result is an immediate status check on your PR. You can\u0026rsquo;t merge until linting passes, which forces clean commits.\nSetting Up the Terraform Workflow # Now for the main event: the workflow that provisions VMs on Proxmox. This pipeline plans on pull requests and applies on pushes to the master branch. All secrets are fetched dynamically from Vault.\nKey features:\nPlan on PR, Apply on Merge: Keeps the master branch as the single source of truth for applied infrastructure. Isolated State: Uses environment-specific state files (dev.tfstate, prod.tfstate). Destroy Mode: Supports teardown from CI via a repository variable. Vault-Driven: No long-lived credentials in the repo. Here\u0026rsquo;s the job header from .github/workflows/terraform-ansible.yml. The permissions block is critical. That id-token: write permission is what allows the workflow to request a JWT from GitHub\u0026rsquo;s OIDC provider, which it then presents to Vault.\nterraform-plan-apply: runs-on: ubuntu-latest permissions: contents: read # Required to checkout code id-token: write # Required to authenticate pull-requests: write # Required to post comments on PRs defaults: run: shell: bash working-directory: ./terraform-provision Step-by-Step Breakdown\n1. Initial Setup # - name: Connect to Tailscale uses: tailscale/github-action@v2 with: oauth-client-id: ${{ secrets.TS_OAUTH_CLIENT_ID }} oauth-secret: ${{ secrets.TS_OAUTH_SECRET }} tags: tag:ci - name: Setup Terraform uses: hashicorp/setup-terraform@v3 with: terraform_version: 1.6.6 This gives the runner access to my homelab via Tailscale. It also installs the required Terraform version. 2. Dynamic Role Selection # - name: Set Vault Role id: set-role run: | if [ \u0026#34;${{ github.event_name }}\u0026#34; == \u0026#34;push\u0026#34; ]; then echo \u0026#34;role=${{ vars.ENV_NAME }}-github-actions-push-role\u0026#34; \u0026gt;\u0026gt; $GITHUB_OUTPUT else echo \u0026#34;role=${{ vars.ENV_NAME }}-github-actions-pr-role\u0026#34; \u0026gt;\u0026gt; $GITHUB_OUTPUT fi As configured in terraform-admin, PRs and pushes use different roles (*-pr-role vs. *-push-role). This step dynamically selects the correct one. $GITHUB_OUTPUT is used to pass the selected role name to the next step. 3. Authenticate to Vault \u0026amp; Fetch Secrets # - name: Import Secrets from Vault uses: hashicorp/vault-action@v3 with: method: jwt url: ${{ vars.VAULT_ADDR }} role: ${{ steps.set-role.outputs.role }} secrets: | kv/shared/data/minio access_key | AWS_ACCESS_KEY_ID ; kv/shared/data/minio secret_key | AWS_SECRET_ACCESS_KEY ; kv/${{ vars.ENV_NAME }}/data/ssh_ca_public_key public_key | TF_VAR_proxmox_ssh_public_key - name: Import Multiple Proxmox Creds from Vault uses: hashicorp/vault-action@v3 with: method: jwt url: ${{ vars.VAULT_ADDR }} role: ${{ steps.set-role.outputs.role }} secrets: | kv/shared/data/proxmox ** | TF_VAR_proxmox_ ; method: jwt tells the action to perform the OIDC login. role: uses the output from the previous step. The secrets: block is a mapping: vault/path field_name | ENV_VAR_NAME. Minio keys are mapped to AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY for Terraform\u0026rsquo;s S3 backend. The SSH CA public key is exported as TF_VAR_proxmox_ssh_public_key, which injects it into cloud-init. The ** | TF_VAR_proxmox_ ; syntax is a handy shortcut. It fetches all fields in the proxmox secret and prefixes them with TF_VAR_proxmox_ (e.g., api_token becomes TF_VAR_proxmox_api_token). More on how to config this plugin: hashicorp/vault-action A GitHub Action that simplifies using HashiCorp Vault™ secrets as build variables. JavaScript 512 164 4. Terraform Init \u0026amp; Plan (with optional destroy) # - name: Terraform Init run: terraform init --backend-config=\u0026#34;key=${{ vars.ENV_NAME }}.tfstate\u0026#34; - name: Terraform Plan env: TF_VAR_env: ${{ vars.ENV_NAME }} run: | # Check if Destroy mode is enabled PLAN_ARGS=\u0026#34;\u0026#34; if [ \u0026#34;${{ vars.DESTROY }}\u0026#34; == \u0026#34;true\u0026#34; ]; then PLAN_ARGS=\u0026#34;-destroy\u0026#34; fi terraform plan $PLAN_ARGS -var-file=\u0026#34;env/${{ vars.ENV_NAME }}/main.tfvars\u0026#34; -out .planfile terraform init uses the AWS creds from Vault to connect to the Minio backend. --backend-config=\u0026quot;key=...\u0026quot; selects the environment-specific state file (e.g., dev.tfstate). A DESTROY repository variable can be set to \u0026ldquo;true\u0026rdquo; to run a destroy plan. -out .planfile saves the exact plan to be used in the apply step. (Note: You\u0026rsquo;ll need to configure variables like ENV_NAME and DESTROY in your repo\u0026rsquo;s Actions settings.)\n5. Post Plan as PR Comment # - name: Post PR comment uses: borchero/terraform-plan-comment@v2 if: github.event_name == \u0026#39;pull_request\u0026#39; with: token: ${{ github.token }} working-directory: \u0026#34;./terraform-provision\u0026#34; planfile: .planfile This step only runs for PRs and posts the plan output as a comment, allowing for review right in the PR. 6. Terraform Apply (Guarded) # - name: Terraform Apply if: github.ref == \u0026#39;refs/heads/master\u0026#39; \u0026amp;\u0026amp; github.event_name == \u0026#39;push\u0026#39; env: TF_VAR_env: ${{ vars.ENV_NAME }} run: terraform apply -auto-approve .planfile This is the gate. The apply step only runs on pushes to the master branch (i.e., after a PR is merged), not on PRs themselves. This maintains a strict plan/apply separation.\nSetting Up the Ansible Workflow # Now that the terraform-plan-apply job has provisioned the VMs, this job bootstraps the Kubernetes cluster on them.\nHere\u0026rsquo;s the job header from terraform-ansible.yml. Notice the conditions:\nneeds: terraform-plan-apply: This is critical. It ensures Ansible only runs after the Terraform job successfully finishes and the VMs actually exist. if: ...: This clause prevents the Ansible job from running during PRs (which are plan-only) or during a DESTROY run. ansible-bootstrap: runs-on: ubuntu-latest needs: terraform-plan-apply # Only run if on master, is a push, AND destroy mode is NOT enabled if: github.ref == \u0026#39;refs/heads/master\u0026#39; \u0026amp;\u0026amp; github.event_name == \u0026#39;push\u0026#39; \u0026amp;\u0026amp; vars.DESTROY != \u0026#39;true\u0026#39; permissions: contents: read # Required to checkout code id-token: write # Required to authenticate Step-by-Step Breakdown\n1. Network and Vault Role # This part is identical to the Terraform job: it connects to Tailscale and uses the set-role step to dynamically choose the *-push-role.\n2. Fetch Ansible Secrets from Vault # This step is similar, but with one critical difference: outputToken: true.\n- name: Import Secrets from Vault uses: hashicorp/vault-action@v3 id: vault_auth with: method: jwt url: ${{ vars.VAULT_ADDR }} role: ${{ steps.set-role.outputs.role }} outputToken: true secrets: | kv/shared/data/cloudflare * | SSL_ ; kv/${{vars.ENV_NAME}}/data/ip * | IP_ ; kv/${{vars.ENV_NAME}}/data/rke2 * | RKE2_ ; outputToken: true: This is the key. Unlike the Terraform job, we need the raw Vault token as an output (steps.vault_auth.outputs.vault_token) so we can use the Vault CLI in the next step to sign our SSH key. The * | SSL_ wildcard syntax is a clean way to map all fields in a secret (e.g., API_TOKEN in kv/shared/data/cloudflare) to environment variables with a prefix (e.g., SSL_API_TOKEN). I then updated my Ansible group_vars to read these new environment variables.\nansible/inventory/group_vars/all.yaml\nansible_user: ubuntu rke2_version: \u0026#34;v1.32.3+rke2r1\u0026#34; arch: amd64 # type of machine, raspberry pi use arm64 rke2_token: \u0026#34;{{ lookup(\u0026#39;env\u0026#39;, \u0026#39;RKE2_TOKEN\u0026#39;) }}\u0026#34; # for authenticate \u0026amp; add nodes in cluster vip: \u0026#34;{{ lookup(\u0026#39;env\u0026#39;, \u0026#39;IP_VIP\u0026#39;) }}\u0026#34; # for virtual ip of the servers vip_cidr: \u0026#34;{{ lookup(\u0026#39;env\u0026#39;, \u0026#39;IP_CIDR\u0026#39;) }}\u0026#34; vip_lb_range: \u0026#34;{{ lookup(\u0026#39;env\u0026#39;, \u0026#39;IP_LB_RANGE\u0026#39;) }}\u0026#34; # load balancer ip range vip_ingress_ip: \u0026#34;{{ lookup(\u0026#39;env\u0026#39;, \u0026#39;IP_INGRESS\u0026#39;) }}\u0026#34; # default traefik ip, must be in the range above ssl_local_domain: \u0026#34;{{ env }}.{{ lookup(\u0026#39;env\u0026#39;, \u0026#39;SSL_DOMAIN\u0026#39;) }}\u0026#34; ssl_cloudflare_api_token: \u0026#34;{{ lookup(\u0026#39;env\u0026#39;, \u0026#39;SSL_API_TOKEN\u0026#39;) }}\u0026#34; ssl_email: \u0026#34;{{ lookup(\u0026#39;env\u0026#39;, \u0026#39;SSL_EMAIL\u0026#39;) }}\u0026#34; 3. Generate and Sign Ephemeral SSH Keys # This is the core of the Zero Trust SSH flow. We install the Vault CLI, generate a new keypair for this job, and have Vault sign the public key.\n- uses: eLco/setup-vault@v1 with: vault_version: 1.21.0 - name: Generate and Sign SSH Key env: VAULT_ADDR: ${{ vars.VAULT_ADDR }} VAULT_TOKEN: ${{ steps.vault_auth.outputs.vault_token }} run: | # Generate a fresh SSH keypair locally (no passphrase) ssh-keygen -t rsa -b 4096 -f ./runner_key -q -N \u0026#34;\u0026#34; # Send the Public Key to Vault for signing vault write -field=signed_key ${{vars.ENV_NAME}}-ssh-client-signer/sign/github-runner \\ public_key=@./runner_key.pub \\ valid_principals=ubuntu \u0026gt; runner_key-cert.pub # Set strict permissions chmod 600 runner_key chmod 644 runner_key-cert.pub The private key is generated on the runner and is destroyed when the job finishes. It never leaves the runner. We use the Vault token from the previous step to authenticate the vault write command. Vault signs the public key using the environment-specific SSH CA (dev-ssh-client-signer) and the github-runner role. The resulting certificate (runner_key-cert.pub) is short-lived (30 minutes, as configured in terraform-admin). 4. Build Inventory from Terraform Outputs # - name: Generate Ansible Inventory run: | ../scripts/generate-all-hosts.sh ${{ vars.ENV_NAME }} cat ./inventory/hosts.ini This script (which I wrote) consumes the k8s_nodes.json output from Terraform and builds a standard hosts.ini file for Ansible to use. cat the file to the log, which is a lifesaver for debugging CI. 5. Install Ansible and Collections \u0026amp; Run the Playbook # - name: Install Ansible shell: bash run: | sudo apt update sudo apt install -y ansible - name: Install Ansible collections run: ansible-galaxy install -r requirements.yml - name: Run playbook run: ansible-playbook -e \u0026#34;env=${{ vars.ENV_NAME }}\u0026#34; --private-key runner_key site.yaml This is the final step. Ansible uses the ephemeral, signed SSH key to connect and run the cluster bootstrap.\n--private-key runner_key: Ansible automatically finds and uses the corresponding certificate (runner_key-cert.pub). The playbook then runs all the roles (downloading RKE2, adding servers and agents, applying kube-vip and Longhorn, etc.) and successfully bootstraps the cluster. This pipeline is a complete example of Zero Trust principles in action: short-lived identities, scoped policies, and ephemeral network access. The moment you see vault-action successfully mint a token and vault write return a signed SSH cert, that\u0026rsquo;s the \u0026ldquo;JWT authentication pays off\u0026rdquo; moment.\nSummary \u0026amp; next steps # So, as you can see, this is my longest post to date. Migrating the homelab automation to GitHub Actions + HashiCorp Vault wasn\u0026rsquo;t a single leap but a series of decisions about trust, scope, and operational burden.\nWhat I gained:\nA repeatable, observable CI/CD pipeline that provisions and configures infrastructure without manual secrets. A much tighter blast-radius through policy separation and ephemeral credentials. An extensible platform: adding new secret engines (like transit for encryption or database for dynamic DB creds) is now straightforward. What still needs work (Next Steps):\nAutomate Vault Unseal: Move to an auto-unseal backend (like a cloud KMS) or deploy Vault in a high-availability Kubernetes cluster. Add Runtime Tests: Implement smoke tests in the pipeline to probe Kubernetes endpoints after deployment to verify health. Dynamic kubectl Access: Since Vault is now an OIDC provider, I can integrate it with Kubernetes (and maybe Keycloak) to grant dynamic, short-lived kubectl access instead of using static kubeconfig files. Yeah, it was a lot of work. But once it clicks, the day-to-day overhead drops sharply - and you\u0026rsquo;ll stop having to remember which machine had the old SSH key.\nHappy automating.\n","date":"17 November 2025","externalUrl":null,"permalink":"/posts/gitops-github-actions-terraform-ansible/","section":"Posts","summary":"This is Part 2 of the Hashicorp Vault series. Part 1 covered bootstrapping and configuring HashiCorp Vault (the one-time terraform-admin work) and the design goals for Zero Trust secrets management.\n","title":"Vault (Part 2): Secure On-Prem Automation with Ephemeral Runners and Tailscale","type":"posts"},{"content":"In the last two parts of the On-Premise 101 series, I showed you how to provision and bootstrap a full Kubernetes cluster. Now, I\u0026rsquo;m tearing part of it down and rebuilding the entire workflow. Why? The project\u0026rsquo;s goal was always automation, but I hit a wall. The manual steps to set up that automation - like managing secrets and SSH keys - became a huge liability.\nSo, before I show you how to deploy applications on that cluster, I\u0026rsquo;m pivoting. I\u0026rsquo;ve decided to migrate the whole project to use GitHub Actions for CI/CD and HashiCorp Vault for secrets. The idea was simple: push code, run automation in a repeatable environment, fetch secrets from a central server, and never bake a long-lived credential into the repo.\nWhile that sounds great in theory, implementing it was harder than it sounded. Way harder.\nHowever, after a couple of days of wrestling with HashiCorp Vault, I\u0026rsquo;m ready to share my design. In this post, I\u0026rsquo;ll show you my Vault setup, how GitHub Actions authenticates without static tokens, the new Terraform/Ansible SSH flow, and the operational lessons I learned. This is the foundation of a Zero-Trust design.\nIn case anyone haven\u0026rsquo;t read it, here\u0026rsquo;s my blog post on the previous version of the project\nphuchoang2603/kubernetes-proxmox auto provision k8s to proxmox using terraform + ansible + github actions + vault HCL 28 3 From Fragile Keys to Zero Trust # In my previous setup, I manually managed secrets. SSH keys were on my local machine, and other credentials were scattered in config files, all carefully managed with .gitignore. This seemed fine\u0026hellip; until my admin laptop died a few weeks ago.\nSuddenly, I was completely locked out of my own cluster and had to re-bootstrap the entire thing. That single point of failure was a wake-up call.\nThis failure highlighted other problems, too. When reproducing the dev and prod environments, I\u0026rsquo;d hit issues like \u0026ldquo;that one box had a different key\u0026rdquo; or \u0026ldquo;I forgot to update the inventory.\u0026rdquo; And what if I wanted to collaborate? I\u0026rsquo;d have to hand out my static SSH keys. If that person left the project, how could I be sure they wouldn\u0026rsquo;t access my cluster again?\nThat experience changed my thinking from \u0026ldquo;How do I stop losing keys?\u0026rdquo; to \u0026ldquo;How do I design a system where one mistake doesn\u0026rsquo;t burn everything down?\u0026rdquo; Zero Trust became that guiding mindset.\nBy definition, Zero Trust assumes nothing is trusted by default. Every request, credential, and runner must prove who they are. Think less \u0026ldquo;giving someone a copy of the master key\u0026rdquo; and more \u0026ldquo;a bouncer who checks your ID every time you try to get in.\u0026rdquo; That bouncer doesn\u0026rsquo;t rely on a single badge that could be copied; they verify identity, scope, and time.\nIn contrast, the traditional model is fragile: bake a bunch of static credentials into CI/CD configs, hand out long-lived keys, and hope you rotate them before they leak. When a leak does happen, it\u0026rsquo;s hard to trace. With Zero Trust, you flip the model: authenticate identities, minimize privileges, and shrink the lifetime of credentials so a leak is both noisy and short-lived.\nHow Vault and GitHub Actions Map to Zero Trust: # Never trust, always verify: GitHub Actions authenticates to Vault (using OIDC/JWT in our case), and Vault validates the token\u0026rsquo;s claims before issuing anything. It requests dynamic tokens, not hard-coded secrets. Least privilege: Vault policies are explicit. Roles can only read the paths they need. A runner for the dev environment cannot read prod/* secrets, even if it somehow gets a token. Short-lived credentials: Signed SSH certificates and temporary Vault tokens mean credentials expire quickly, reducing the blast radius. Once a job finishes, the environment is destroyed, limiting the persistence of any secret material. Centralized control and audit logging: Vault is the single source of truth for secrets and logs every access. If a leak happens, you can easily identify the source and mitigate it. Why Not \u0026ldquo;Just use GitHub Secrets\u0026rdquo;? # A fair question. For a small homelab, you could store a few secrets in GitHub and be done. The tradeoffs are:\nAny repo collaborator with write permissions can add a workflow to exfiltrate secrets. Static secrets are long-lived unless you rotate them; rotation is manual and error-prone. Policies, CA signing, and TTLs are hard to model in GitHub alone. Vault gives you fine-grained policies, TTLs, and an auditable central place for secrets. The only secret I now store in GitHub is my Tailscale credential, which lets the GitHub Actions runner access my homelab network to talk with Vault and the rest of the infrastructure. (You might not even need this if you run a self-hosted runner, but I prefer Tailscale because I get to use the free compute resources GitHub provides :D).\nThis mindset directly addressed my prior pain. Instead of juggling SSH keys across machines and wondering which config file held a password, I\u0026rsquo;m now designing flows that assume compromise is possible and make it costly for an attacker. That philosophical shift - designing for minimal trust and minimal time windows - sets the foundation for the concrete Terraform and Ansible flows I\u0026rsquo;ll describe next.\nArchitecture Overview # It\u0026rsquo;s not simple to just spin up a Vault server, plug it into GitHub Actions, and call it a day. At first, I planned to use Terraform to automate my Vault setup - after all, Vault is infrastructure, and I want to automate its provisioning.\nHowever, I immediately ran into a classic \u0026ldquo;chicken-and-egg\u0026rdquo; problem. Actually, two of them:\nThe Backend Deadlock: My main Terraform project uses Minio (S3) for its remote state. It needs to fetch Minio credentials from Vault just to initialize. But how can it fetch secrets from Vault before it\u0026rsquo;s even initialized? The Auth Deadlock: I want GitHub Actions to run that Terraform code. But for that, the runner needs to authenticate to Vault using OIDC. Someone has to set up that OIDC auth method and role in Vault first. My GitHub runner can\u0026rsquo;t configure its own access if it can\u0026rsquo;t log in. Therefore, my solution was to split responsibilities into two distinct Terraform projects:\nterraform-admin: This is the foundation, run once from your local machine. It bootstraps Vault, configures auth methods (like OIDC for GitHub), sets up core policies, seeds initial secrets, and configures the SSH Certificate Authority (CA). terraform-provision: This is the main CI/CD project, run by GitHub Actions. It authenticates to the now-configured Vault, fetches the secrets and certs it needs, and then provisions the actual infrastructure (Proxmox VMs, networking, etc.) before handing off to Ansible. Think of terraform-admin as the one-time, manual step to build and secure the \u0026ldquo;secrets vault\u0026rdquo; itself. Once that\u0026rsquo;s done, terraform-provision is the fully automated workflow that just uses the vault.\nAt a glance, the new workflow looks like this: There are a few other pieces worth calling out:\nTailscale: This creates a secure network path for the GitHub runner to access Vault and Proxmox, since the runner isn\u0026rsquo;t on my local LAN. SSH Certificate Authority (Vault): We\u0026rsquo;re now signing short-lived client certificates for SSH access. This completely replaces the need to manage static authorized_keys files on servers. Shared Secrets: A small set of secrets (like my Cloudflare API key) are stored in a shared/ path in Vault, accessed by CI jobs with very strict policies. With that background, let\u0026rsquo;s dive into how to implement this.\nSetting up HashiCorp Vault # Since my terraform-admin project needs secrets (like Minio credentials) just to initialize its remote state, the Vault server has to exist before the admin automation can run.\nThis means I need to manually set up the Vault server and seed it with those critical, first-step secrets. Only then can I run terraform-admin to automate setting up all the other policies, auth methods, and engines.\nHere\u0026rsquo;s the manual bootstrap process:\nDeploy Vault as a container (Docker Compose) on a dedicated VM in my Proxmox cluster. Initialize (unseal) Vault and securely store the initial root token and unseal keys. Design the secret structure and manually populate the absolute minimum critical secrets (like the Minio credentials) via the Vault UI. Run terraform-admin (which can now fetch the Minio credentials) to automate the rest. Deploying Vault # Best practice would be to run Vault on Kubernetes, but I wanted to get this working with my project first. I decided to run Vault as a Docker container using docker-compose. It\u0026rsquo;s simple, reproducible, and I can easily migrate to a K8s deployment later.\ndocker-compose.yaml\nservices: vault: image: hashicorp/vault:1.19.0 container_name: vault command: server restart: unless-stopped cap_add: - IPC_LOCK volumes: - /mnt/storage/appdata/vault:/vault/file - ./config:/vault/config The Vault configuration lives in ./config/local.hcl and is mounted into the container.\nconfig/local.hcl\n# Configure the \u0026#39;file\u0026#39; storage backend storage \u0026#34;file\u0026#34; { path = \u0026#34;/vault/file\u0026#34; } # Configure the listener listener \u0026#34;tcp\u0026#34; { address = \u0026#34;0.0.0.0:8200\u0026#34; tls_disable = true } # Add the api_addr to prevent warnings api_addr = \u0026#34;http://192.168.1.10:8200\u0026#34; # Use your VM\u0026#39;s static IP # Other settings ui = true disable_mlock = false A few key decisions here:\nTLS is disabled: This is because I run a separate Nginx reverse proxy in front of Vault to handle all TLS termination. File storage backend: This is perfectly fine for a single-node homelab. It\u0026rsquo;s simple, and the data at rest is still encrypted by Vault. cap_add: IPC_LOCK: This is critical. It allows Vault to lock its process memory (mlock) and prevents the operating system from swapping sensitive secrets to disk. First-time initialization (Shamir\u0026rsquo;s Secret Sharing) # When Vault starts for the first time, you have to initialize it. This process creates the master encryption key, splits it into \u0026ldquo;unseal keys\u0026rdquo; (using Shamir\u0026rsquo;s Secret Sharing), and generates the initial root token.\nThere are two common approaches:\nApproach 1: The Solo Homelab (What I Use) Key shares: 1 Key threshold: 1 Result: This gives you one unseal key and one root token. It\u0026rsquo;s the easiest to manage for a personal project, but you are 100% responsible for storing that one unseal key safely. Approach 2: The High-Security Team Key shares: 5 Key threshold: 3 Result: Vault creates 5 unique key shares, and any 3 of them are required to unseal. This prevents any single operator from having full control and is resilient if two people are on vacation. Designing the Secrets Structure (Environment-First) # Early on, I made the mistake of organizing secrets by tool (e.g., terraform/dev, ansible/dev). I quickly found it\u0026rsquo;s much easier to reason about policies and blast radius when secrets are grouped by environment first:\nkv/data/dev/ kv/data/prod/ kv/data/shared/ (For cross-environment credentials: Cloudflare API key, Minio admin credentials, etc.) This \u0026ldquo;environment-first\u0026rdquo; design has two huge benefits:\nLeast-Privilege Policies: It\u0026rsquo;s straightforward to write policies that map to this structure. For example, a dev policy can be allowed read access to kv/data/dev/* and kv/data/shared/* but be explicitly denied from kv/data/prod/*. Blast Radius: If a dev pipeline or runner is compromised, it simply cannot read production secrets. I populate these initial, critical secrets manually via the Vault UI. From that point on, I can use the Vault CLI to log in, fetch those Minio credentials, and let terraform-admin automate everything else.\nThis is where terraform-admin earns its keep. It bootstraps the critical Vault components that GitHub Actions will rely on. The goal is a true Zero-Trust workflow:\nRunners prove who they are using short-lived JWTs (OIDC). They get only the secrets they need via environment-specific policies. They get ephemeral SSH access by requesting temporary certificates from Vault\u0026rsquo;s CA. Creating Policies and JWT/OIDC Auth for GitHub Actions # What I Want to Achieve # Environment-specific policies (dev-policy, prod-policy) that can only read their own kv paths. A shared-policy for common, cross-environment secrets (like Cloudflare or Minio keys). A single JWT/OIDC auth backend that trusts GitHub\u0026rsquo;s OIDC provider. JWT roles that map a specific GitHub repo, branch, or PR to a set of Vault policies. This diagram shows the authentication flow when a GitHub Actions runner starts:\nThe runner asks GitHub\u0026rsquo;s OIDC provider for a short-lived JWT. The provider, validating the runner\u0026rsquo;s context, issues a signed JWT. The runner presents this JWT to Vault\u0026rsquo;s JWT auth backend. Vault validates the JWT\u0026rsquo;s signature against GitHub\u0026rsquo;s public keys and checks the token\u0026rsquo;s claims (like issuer, audience, and sub for the repo/branch) against the bound_claims configured in the Vault role. If everything matches, Vault mints a temporary Vault token that has the correct policies attached. The runner now uses this short-lived Vault token to read secrets and request SSH certificates.\nOverview of the implementation # Enough talking, let\u0026rsquo;s dive into the code. The terraform-admin project is structured as a root main.tf that calls a reusable module.\n. ├── main.tf ├── mise.toml ├── modules │ └── vault-admin │ ├── main.tf │ ├── outputs.tf │ └── variables.tf ├── outputs.tf └── provider.tf First, in the root main.tf, we enable a shared JWT auth backend that only trusts tokens from GitHub Actions.\n# Shared JWT Auth Backend (used by both dev and prod) resource \u0026#34;vault_jwt_auth_backend\u0026#34; \u0026#34;jwt\u0026#34; { path = \u0026#34;jwt\u0026#34; bound_issuer = \u0026#34;https://token.actions.githubusercontent.com\u0026#34; oidc_discovery_url = \u0026#34;https://token.actions.githubusercontent.com\u0026#34; } What this does:\nbound_issuer: Tells Vault to only accept tokens issued by token.actions.githubusercontent.com. oidc_discovery_url: The URL where Vault can fetch GitHub\u0026rsquo;s public keys to verify the JWT signatures. A Debugging War Story: We must share this backend and, crucially, it must be mounted at the path jwt. I spent hours debugging this because I first tried a custom path (github-oidc).\nIt turns out the official hashicorp/vault-action plugin is hard-coded to use the default jwt path.\nMy GitHub Action was sending login attempts to: .../v1/auth/jwt/login My Vault roles were configured and waiting at: .../v1/auth/github-oidc/login Vault was correctly rejecting the login at the jwt path because there were no roles there. Save yourself the headache: just use path = \u0026quot;jwt\u0026quot;.\nNext, we define a small shared policy for credentials used across environments (like the Cloudflare API key or Minio admin creds).\n# Shared Policy (used by both dev and prod) resource \u0026#34;vault_policy\u0026#34; \u0026#34;shared_policy\u0026#34; { name = \u0026#34;shared-policy\u0026#34; policy = \u0026lt;\u0026lt;-EOT # Shared policy path \u0026#34;kv/shared/data/*\u0026#34; { capabilities = [\u0026#34;read\u0026#34;, \u0026#34;list\u0026#34;] } EOT } Now, we use a module to create the environment-specific configurations. This keeps the behavior consistent for dev and prod.\n# Dev Environment module \u0026#34;vault_admin_dev\u0026#34; { source = \u0026#34;./modules/vault-admin\u0026#34; env = \u0026#34;dev\u0026#34; jwt_backend_path = vault_jwt_auth_backend.jwt.path shared_policy_name = vault_policy.shared_policy.name github_organization = \u0026#34;phuchoang2603\u0026#34; github_repository = \u0026#34;kubernetes-proxmox\u0026#34; github_branch = \u0026#34;master\u0026#34; } # Prod Environment module \u0026#34;vault_admin_prod\u0026#34; { source = \u0026#34;./modules/vault-admin\u0026#34; env = \u0026#34;prod\u0026#34; jwt_backend_path = vault_jwt_auth_backend.jwt.path shared_policy_name = vault_policy.shared_policy.name github_organization = \u0026#34;phuchoang2603\u0026#34; github_repository = \u0026#34;kubernetes-proxmox\u0026#34; github_branch = \u0026#34;master\u0026#34; } Inside the Module (modules/vault-admin/main.tf) # Diving into the module, this is where we create the roles that bind a GitHub Action run to a set of policies. We create two roles: one for push events and one for pull requests.\n# GitHub Actions Push Role resource \u0026#34;vault_jwt_auth_backend_role\u0026#34; \u0026#34;github_actions_push_role\u0026#34; { backend = var.jwt_backend_path role_name = \u0026#34;${var.env}-github-actions-push-role\u0026#34; role_type = \u0026#34;jwt\u0026#34; token_policies = [vault_policy.vault_env_policy.name, var.shared_policy_name] bound_audiences = [\u0026#34;https://github.com/${var.github_organization}\u0026#34;] bound_claims_type = \u0026#34;glob\u0026#34; bound_claims = { \u0026#34;sub\u0026#34; = \u0026#34;repo:${var.github_organization}/${var.github_repository}:ref:refs/heads/${var.github_branch}\u0026#34; } user_claim = \u0026#34;actor\u0026#34; } backend: Points to the shared jwt auth backend. role_name: A unique name, like dev-github-actions-push-role. token_policies: Important. It attaches both the environment-specific policy (dev-policy) and the shared-policy. bound_audiences: Must match the JWT\u0026rsquo;s aud claim. bound_claims: Critical security control. It constrains the JWT sub (subject) claim. Only tokens minted for this exact repo:org/repo:ref:refs/heads/branch string can use this role. user_claim: Maps the JWT actor (the GitHub user who triggered the run) to the Vault identity for auditing. A second role for pull requests looks similar but accepts pull_request subjects.\n# GitHub Actions PR Role resource \u0026#34;vault_jwt_auth_backend_role\u0026#34; \u0026#34;github_actions_pr_role\u0026#34; { backend = var.jwt_backend_path role_name = \u0026#34;${var.env}-github-actions-pr-role\u0026#34; role_type = \u0026#34;jwt\u0026#34; token_policies = [vault_policy.vault_env_policy.name, var.shared_policy_name] bound_audiences = [\u0026#34;https://github.com/${var.github_organization}\u0026#34;] bound_claims_type = \u0026#34;glob\u0026#34; bound_claims = { \u0026#34;sub\u0026#34; = \u0026#34;repo:${var.github_organization}/${var.github_repository}:pull_request\u0026#34; } user_claim = \u0026#34;actor\u0026#34; } Why separate PR and push roles? You could use different policies, for example, giving PRs read-only access for terraform plan jobs. But to be honest, I initially wanted to group them and couldn\u0026rsquo;t find a clean way to do it, so separate roles it is.\nHere\u0026rsquo;s the environment-specific policy referenced in token_policies.\n# Environment-specific Vault Policy resource \u0026#34;vault_policy\u0026#34; \u0026#34;vault_env_policy\u0026#34; { name = \u0026#34;${var.env}-policy\u0026#34; policy = \u0026lt;\u0026lt;-EOT path \u0026#34;kv/${var.env}/data/*\u0026#34; { capabilities = [\u0026#34;read\u0026#34;, \u0026#34;list\u0026#34;] } # Grant permission to sign keys using a specific role path \u0026#34;${vault_mount.ssh_client_signer.path}/sign/github-runner\u0026#34; { capabilities = [\u0026#34;update\u0026#34;] } EOT } This policy does two things:\nCreates dev-policy or prod-policy that only allows reading from its own kv path (e.g., kv/dev/data/*). This enforces true isolation. It grants permission for the runner to request an SSH certificate from the SSH engine. Let\u0026rsquo;s talk about that next. SSH Certificate Authority for Ephemeral Access # Why This Exists # In the previous project, I configured terraform-provision to inject my static, local public SSH key into the VM using cloud-init. This was necessary for Ansible to connect.\nWhen porting to GitHub Actions, keeping a long-lived SSH private key in GitHub Secrets is a huge risk.\nThe solution is to use Vault\u0026rsquo;s SSH engine as a Certificate Authority (CA). The runners will generate a new, ephemeral SSH key for each job, ask Vault to sign the public key, and then use that short-lived certificate to log in. The VMs will be configured to trust any certificate signed by our Vault CA.\nThis flow is far more secure:\nDuring provisioning (terraform-provision), the Vault CA\u0026rsquo;s public key is injected into each VM\u0026rsquo;s cloud-init config. sshd is told to trust this CA. During a workflow run, the GitHub runner generates a fresh SSH keypair. The runner authenticates to Vault (using its JWT, as described above) and asks Vault to sign its new public key. Vault verifies the runner\u0026rsquo;s token and policies, then signs the key, returning a certificate valid for a short TTL (e.g., 30 minutes). The runner uses its private key + the signed certificate to run Ansible. Once the TTL expires, the certificate is useless. Continued at modules/vault-admin/main.tf # First, we mount an SSH secrets engine for each environment.\n# SSH Client Signer Mount resource \u0026#34;vault_mount\u0026#34; \u0026#34;ssh_client_signer\u0026#34; { type = \u0026#34;ssh\u0026#34; path = \u0026#34;${var.env}-ssh-client-signer\u0026#34; } Next, we tell Vault to generate a CA keypair on that engine. The private key never leaves Vault.\n# SSH CA Configuration resource \u0026#34;vault_ssh_secret_backend_ca\u0026#34; \u0026#34;ssh_ca\u0026#34; { backend = vault_mount.ssh_client_signer.path generate_signing_key = true } This next part is the critical handoff to terraform-provision. We store the CA\u0026rsquo;s public key in a K/V secret of the Vault server. The terraform-provision project, when run on Github Actions, will read this secret and inject it into cloud-init.\n# Store SSH CA Public Key resource \u0026#34;vault_generic_secret\u0026#34; \u0026#34;ssh_ca_public_key\u0026#34; { path = \u0026#34;kv/${var.env}/ssh_ca_public_key\u0026#34; data_json = jsonencode({ public_key = vault_ssh_secret_backend_ca.ssh_ca.public_key }) } Finally, we define the signing role that the GitHub runners will use. This role, github-runner, is what the JWT-authenticated token is allowed to access.\nkey_type = \u0026quot;ca\u0026quot;: Indicates we want Vault to sign client keys using its CA. allow_user_certificates = true: We are signing certificates for users (clients), not hosts. allowed_users = \u0026quot;ubuntu\u0026quot;: A key security constraint. The resulting certificate will only be valid for logging in as the ubuntu user, which helps reduce privilege escalation. ttl = \u0026quot;1800\u0026quot;: The certificate expires after 30 minutes, ensuring access is ephemeral. # SSH Role for GitHub Runner resource \u0026#34;vault_ssh_secret_backend_role\u0026#34; \u0026#34;github_runner\u0026#34; { backend = vault_mount.ssh_client_signer.path name = \u0026#34;github-runner\u0026#34; key_type = \u0026#34;ca\u0026#34; allow_user_certificates = true allowed_users = \u0026#34;ubuntu\u0026#34; ttl = \u0026#34;1800\u0026#34; # 30 minutes } The SSH Signing Flow in Action # In the GitHub Actions workflow, the runner will perform these steps for every job:\n# 1. Generate a new, ephemeral keypair for this job ssh-keygen -t rsa -b 4096 -f ./runner_key -q -N \u0026#34;\u0026#34; # 2. Ask Vault to sign the new public key vault write -field=signed_key dev-ssh-client-signer/sign/github-runner \\ public_key=@./runner_key.pub \\ valid_principals=\u0026#34;ubuntu\u0026#34; \u0026gt; runner_key-cert.pub Ansible then uses this keypair (runner_key and runner_key-cert.pub) to connect. The remote VM verifies the certificate, checks that it was signed by the trusted Vault CA (the public key we injected via cloud-init), and grants access.\nThis is where all the pieces click together. Remember that environment-specific policy (dev-policy) from before? It had this path:\npath \u0026quot;${vault_mount.ssh_client_signer.path}/sign/github-runner\u0026quot; { capabilities = [\u0026quot;update\u0026quot;] }\nWithout that permission, the runner\u0026rsquo;s JWT-authed token would be valid but powerless to request a certificate. This is the perfect example of how the JWT auth role (who the runner is), the policy (what the runner can do), and the SSH engine (the action itself) work together to provide short-lived, auditable, and secure access.\nRunning terraform-admin # To run this, we need a provider.tf for the terraform-admin project itself.\n# provider.tf terraform { required_version = \u0026#34;\u0026gt;= 1.6.6\u0026#34; required_providers { vault = { source = \u0026#34;hashicorp/vault\u0026#34; version = \u0026#34;5.0.0\u0026#34; } } backend \u0026#34;s3\u0026#34; { bucket = \u0026#34;terraform\u0026#34; key = \u0026#34;admin.tfstate\u0026#34; region = \u0026#34;us-east-1\u0026#34; endpoints = { s3 = \u0026#34;http://10.69.1.102:9000\u0026#34; } skip_credentials_validation = true skip_metadata_api_check = true skip_region_validation = true skip_requesting_account_id = true use_path_style = true } } provider \u0026#34;vault\u0026#34; { # The Vault address and token must be set in the VAULT_ADDR and VAULT_TOKEN environment variable. } The S3 backend (Minio) stores the state for this admin project. This is why we had to manually create the Minio secrets in Vault first. We\u0026rsquo;ll fetch those Minio credentials and export them as AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY so Terraform can initialize. The Vault provider reads VAULT_ADDR and your initial VAULT_TOKEN (the root token) from the environment. # Written in Nushell, adjust for Bash if needed # 1. Set Vault connection details (use your root token for this first run) $env.VAULT_ADDR = \u0026#34;https://vault.home.phuchoang.sbs\u0026#34; $env.VAULT_TOKEN = \u0026#34;\u0026lt;your-initial-root-token\u0026gt;\u0026#34; # 2. Fetch Minio creds for the Terraform backend $env.AWS_ACCESS_KEY_ID = (vault kv get -field=access_key kv/shared/minio) $env.AWS_SECRET_ACCESS_KEY = (vault kv get -field=secret_key kv/shared/minio) # 3. Run terraform-admin (only needs to be run once) cd terraform-admin terraform init terraform apply After applying, your Vault is fully configured. The JWT auth backend is enabled, policies are created, and the SSH CAs are generated and have their public keys ready for terraform-provision to consume.\nVault is Ready: What\u0026rsquo;s Next? # At this point, your Vault server is fully operational and configured with everything needed for a Zero Trust automation workflow:\nJWT/OIDC authentication is enabled and waiting for GitHub Actions runners to prove their identity. Environment-specific policies ensure dev and prod are completely isolated. SSH Certificate Authorities are ready to sign ephemeral keys for secure VM access. Secrets are organized by environment and ready to be consumed. This foundation makes the entire automation workflow possible. Without it, we\u0026rsquo;d be back to storing long-lived credentials in GitHub Secrets or scattering SSH keys across machines.\nIn the next post, I\u0026rsquo;ll show you how GitHub Actions uses this Vault setup to:\nAuthenticate without storing any static tokens Provision VMs on Proxmox using Terraform Bootstrap a Kubernetes cluster using Ansible with ephemeral SSH certificates Run the entire workflow automatically on every push to master The hard part is done. Now it\u0026rsquo;s time to see it in action.\n","date":"16 November 2025","externalUrl":null,"permalink":"/posts/gitops-github-actions-hashicorp-vault/","section":"Posts","summary":"In the last two parts of the On-Premise 101 series, I showed you how to provision and bootstrap a full Kubernetes cluster. Now, I’m tearing part of it down and rebuilding the entire workflow. Why? The project’s goal was always automation, but I hit a wall. The manual steps to set up that automation - like managing secrets and SSH keys - became a huge liability.\n","title":"Vault (Part 1): HashiCorp Vault for Zero Trust Secrets Management","type":"posts"},{"content":"","date":"11 November 2025","externalUrl":null,"permalink":"/series/on-premise-101/","section":"Series","summary":"","title":"On-Premise 101","type":"series"},{"content":"People are often scared to learn Kubernetes because it\u0026rsquo;s difficult to set up an environment to play around with. Sure, you can use a project like minikube to learn on your own, but running it on your laptop seems limited, and you may not get to experience all the awesome features Kubernetes offers (remember, it was born for high availability). Or, if you have money, you can use a managed Kubernetes cluster from one of the cloud providers. Often, they\u0026rsquo;ve abstracted away so many things under the hood that you only need to learn kubectl and you\u0026rsquo;re good to go.\nHowever, if you encounter issues, you might find it hard to debug those services, likely because you just don\u0026rsquo;t have to touch the internals frequently. I mean, who in their right mind would break things on a cluster frequently and debug it just for the sake of learning?\nWell, I do. But since I already have my own homelab, I decided to experiment on that rather than on a managed cloud cluster.\nWhat\u0026rsquo;s more, this also gives me the option to properly deploy Kubernetes from the ground up. I also want to automate all the steps I\u0026rsquo;ve gone through so that I can reverse something and run it all back if it goes wrong. After creating 12 VMs with Terraform in the previous part, this post is the second part of my deep dive on the project below, focusing on how I use Ansible to automate the deployment of Kubernetes on those VMs.\nphuchoang2603/kubernetes-proxmox auto provision k8s to proxmox using terraform + ansible + github actions + vault HCL 28 3 Video demo:\nMy K8s Flavor # Before we dive in, I have to say that we are not going to bootstrap a whole Kubernetes cluster from the ground up as detailed in the project below. However, I absolutely recommend anyone who wants to learn it from the beginning to go through this project. The reason I\u0026rsquo;m not using it is that it\u0026rsquo;s not recommended for daily usage. Instead, I will use RKE2, which is more production-ready but still offers a great learning experience.\nkelseyhightower/kubernetes-the-hard-way Bootstrap Kubernetes the hard way. No scripts. null 50358 15930 What is RKE2? (according to the documentation) # RKE2 is Rancher\u0026rsquo;s enterprise-ready, next-generation Kubernetes distribution. It\u0026rsquo;s also been known as RKE Government. It is a fully conformant Kubernetes distribution that focuses on security and compliance.\nTo meet these goals, RKE2 does the following:\nProvides defaults and configuration options that allow clusters to pass the CIS Kubernetes Benchmark v1.7 or v1.8 with minimal operator intervention Enables FIPS 140-2 compliance Regularly scans components for CVEs using trivy in our build pipeline How is this different from RKE or K3s? # RKE2 combines the best of RKE1 and K3s.\nFrom K3s, it inherits usability, ease of operation, and a simple deployment model. From RKE1, it inherits close alignment with upstream Kubernetes. Where K3s diverged from upstream to optimize for edge deployments, RKE2 stays closely aligned.\nImportantly, RKE2 does not rely on Docker as RKE1 does. RKE1 used Docker to deploy control plane components. RKE2 launches control plane components as static pods, managed by the kubelet, and uses containerd as its embedded container runtime.\nRKE2 Architecture # Image from RKE2 offical docs\nRKE2 brings together several Open Source technologies to make this work:\nK8s API Server: The central hub. All cluster communication goes through here. Scheduler: Decides which node a new pod should run on. Controller Manager: The \u0026ldquo;thermostat.\u0026rdquo; Works to make the actual state match the desired state (e.g., \u0026ldquo;I need 3 pods\u0026rdquo;). Kubelet: The agent on each node. Takes orders from the API Server and tells containerd what to do. Proxy: A network proxy on each node that manages virtual IPs for services. etcd: The cluster\u0026rsquo;s database. Stores the \u0026ldquo;desired state\u0026rdquo; of everything. CoreDNS: The internal DNS server for the cluster. cri: An API that lets kubelet talk to different container runtimes. containerd: The container runtime that manages the complete container lifecycle. runc: The low-level tool that actually creates and runs the containers CNI: The plugin system for pod networking (RKE2 uses canal by default). Traefik: The default Ingress controller (routes external traffic into the cluster). Metrics Server: Collects resource usage data (CPU/RAM). Helm \u0026amp; Helm Controller: A package manager. The helm-controller can automatically apply any charts found in /var/lib/rancher/rke2/server/manifests. A 30-Second Kubernetes Refresher # To understand Kubernetes objects broadly, here\u0026rsquo;s a concise table:\nCategory Object Purpose \u0026amp; Explanation Workloads Pod The smallest deployable unit. Holds one or more containers that share a network and storage. (Objects that run or manage containers) ReplicaSet Ensures a specific number (\u0026ldquo;X\u0026rdquo;) of identical pods are always running. (You rarely use this directly). Deployment Manages the lifecycle of stateless apps. Defines a desired state and manages rolling updates via ReplicaSets. StatefulSet Manages stateful apps (like databases) by giving each pod a stable, unique identity (e.g., db-0) and storage. DaemonSet Ensures one pod runs on every (or specific) node, typically for node-level agents like logging or monitoring. Job Runs a one-time task to completion by creating pods that run once and then stop. CronJob Runs a Job on a schedule (e.g., a nightly backup). Networking Service Provides a stable network endpoint (IP/name) for a group of pods, acting as an internal load balancer and for service discovery. Storage PersistentVolume (PV) The actual \u0026ldquo;supply\u0026rdquo; of storage (e.g., a cloud disk) made available to the cluster by an administrator. PersistentVolumeClaim (PVC) A user\u0026rsquo;s \u0026ldquo;request\u0026rdquo; for storage that claims an available PersistentVolume (PV). Configuration ConfigMap Stores non-sensitive configuration data (like config files or env variables) as key-value pairs to inject into pods. Secret Stores sensitive data (like passwords, API keys, or certificates) securely, to be injected into pods. Organization Namespace A \u0026ldquo;virtual cluster\u0026rdquo; that isolates resources between different teams, projects, or environments (e.g., dev, prod). To sum up how all of that comes into play, here\u0026rsquo;s a nice diagram of how a pod is created, drawn by rajeshsgr. After introducing RKE2 and some basic Kubernetes knowledge, let\u0026rsquo;s move on to deploying it with Ansible.\nThe Elephant in the Room: Terraform vs. Ansible # Alright, let\u0026rsquo;s address the big elephant in the room. \u0026ldquo;I already have one powerful automation tool like Terraform, why on earth would I add another tool like Ansible?\u0026rdquo;\nIt\u0026rsquo;s a fair question. Both are \u0026ldquo;Infrastructure as Code,\u0026rdquo; right? You could say that, but they don\u0026rsquo;t solve the same core problem. The classic analogy is: Terraform builds the house, and Ansible furnishes it. You need both because they are masters of two different domains: provisioning and configuration.\nLet\u0026rsquo;s be clear on what Terraform rocks at. It\u0026rsquo;s brilliant at creating, updating, and destroying the \u0026ldquo;metal.\u0026rdquo; But once that server is \u0026ldquo;on,\u0026rdquo; Terraform\u0026rsquo;s job is pretty much done. Don\u0026rsquo;t attempt to use it for configuration. Sure, you can use remote-exec provisioners to run shell scripts. But it\u0026rsquo;s awful, hard to debug, and if it fails, Terraform might \u0026ldquo;taint\u0026rdquo; the resource and want to destroy it. It has no logic for \u0026ldquo;rolling updates\u0026rdquo; or complex app deployments.\nThis is where Ansible steps in. Ansible is an automation and configuration management tool. Its job is to take the servers Terraform built and make them do something useful. Ansible is (mostly) procedural. You write a Playbook (in simple YAML) that lists a set of tasks to run, in order. It\u0026rsquo;s agentless (it just uses SSH) and it doesn\u0026rsquo;t care about \u0026ldquo;state\u0026rdquo; in the same way Terraform does. It just runs your tasks.\nMy Automation Workflow: From Terraform to Ansible # Therefore, the best practice is to combine both tools. The workflow to deploy the Kubernetes cluster on Proxmox looks like this:\nPhase 1: Terraform Builds the House (Covered in Part 4) # Download a cloud-init-ready image.\nDefine specs (CPU, memory, disk).\nUse cloud-init to set the static IP and inject your public SSH key.\nCreate the VMs (three types: servers, agents, longhorn nodes). servers are for the control plane, agents are worker nodes, and longhorn nodes are worker nodes configured with lots of storage to act as our high-availability persistent storage.\nThe Handoff: Our \u0026ldquo;Terraform-to-Ansible\u0026rdquo; Glue # My k8s_nodes.json file (covered in the last part) from Terraform is our \u0026ldquo;source of truth.\u0026rdquo; I run a simple script that parses that JSON and auto-generates the inventory/hosts.ini file that Ansible needs. This means if I add a node in Terraform, it automatically gets configured by Ansible on the next run.\n./scripts/generate-all-hosts.sh dev I also run a cleanup script to clear old SSH keys, which prevents \u0026ldquo;host key\u0026rdquo; errors:\n./scripts/clean-up-ssh-known-hosts.sh dev Phase 2: Ansible Furnishes the House # Prepare all nodes (enable IP forwarding, create directories) \u0026amp; Download RKE2.\nBootstrap RKE2 on the servers nodes.\nJoin the agents and longhorn nodes to the cluster.\nDeploy kube-vip for a high-availability VIP for the servers and to provide LoadBalancer IPs for Services.\nDeploy Helm applications dynamically from a centralized configuration (helm.yaml), including:\ncert-manager for SSL certificate management traefik as our ingress controller longhorn for high-availability persistent storage argo-cd for GitOps deployment of all our applications (covered in the next part) Additional apps like cloudnative-pg, external-secrets, and more Fetch the kubectl config from the server to our local machine.\nHere\u0026rsquo;s the overview of my Ansible structure based on that workflow. ansible.cfg sets default CLI arguments. collections/requirements.yaml lists required Ansible plugins. The inventory directory stores our variables (like the VM IPs and Helm app configurations). site.yaml is the main playbook that calls the modular playbooks (Roles) located in the roles directory.\n. ├── ansible.cfg ├── collections │ └── requirements.yaml ├── inventory │ ├── group_vars │ │ └── all │ │ ├── helm.yaml # Centralized Helm apps config │ │ └── main.yaml # Cluster variables │ └── hosts.ini ├── roles │ ├── add_agent │ ├── add_server │ ├── apply_kube_vip │ ├── deploy_helm_apps # Dynamic Helm deployment │ └── download_rke2 └── site.yaml Setting the Stage: Ansible Inventory and Variables # As introduced above, here\u0026rsquo;s the inventory/hosts.ini file. It contains the IPs of the machines that Ansible will target. Notice it has three categories (agents, longhorn, servers) and one big group-of-groups: rke2:children. This is needed to specify whether a task should run on a specific group or on all machines. This file is auto-generated by the script in the handoff phase.\n[agents] dev-agent1 ansible_host=10.69.1.181 dev-agent2 ansible_host=10.69.1.182 [longhorn] dev-longhorn1 ansible_host=10.69.1.114 dev-longhorn2 ansible_host=10.69.1.115 dev-longhorn3 ansible_host=10.69.1.116 [servers] dev-server1 ansible_host=10.69.1.111 dev-server2 ansible_host=10.69.1.112 dev-server3 ansible_host=10.69.1.113 [rke2:children] agents longhorn servers Additionally, inventory/group_vars/all.yaml is our main variables config file.\nansible_user: ubuntu rke2_version: \u0026#34;v1.32.3+rke2r1\u0026#34; arch: amd64 # type of machine, raspberry pi use arm64 rke2_token: dev # for authenticate \u0026amp; add nodes in cluster env: dev vip: \u0026#34;10.69.1.110\u0026#34; # for virtual ip of the servers vip_cidr: \u0026#34;16\u0026#34; vip_lb_range: \u0026#34;10.69.1.117-10.69.1.119\u0026#34; # load balancer ip range ssl_local_domain: dev.phuchoang.sbs ssl_cloudflare_api_token: ssl_email: xuanphuc.a1gv@gmail.com ssl_ingress_ip: \u0026#34;10.69.1.117\u0026#34; # default traefik ip, must be in the range above The Master Plan: Our site.yaml Playbook # As introduced earlier, site.yaml is the master playbook that calls the child roles. It sources the tasks from roles/\u0026lt;role_name\u0026gt;/tasks/main.yaml. Here\u0026rsquo;s the overview of my site.yaml:\n- name: Prepare all nodes \u0026amp; Download RKE2 hosts: rke2 # servers + agents + longhorn nodes combined gather_facts: true roles: - download_rke2 # Bootstrap k8s - name: Bootstrap RKE2 Servers hosts: servers gather_facts: true roles: - add_server - name: Add additional RKE2 agents \u0026amp; longhorn agents hosts: agents, longhorn gather_facts: true roles: - add_agent # Deploy applications - name: Deploy applications hosts: servers gather_facts: true run_once: true roles: - role: apply_kube_vip - role: deploy_helm_apps # Deploys all Helm apps from helm.yaml Notice how much simpler this is compared to having separate roles for each application. Instead of calling apply-ssl, apply-longhorn, and apply-argocd individually, we now have a single deploy_helm_apps role that reads all application definitions from inventory/group_vars/all/helm.yaml and deploys them dynamically. This makes adding or removing applications as simple as editing a YAML configuration file.\nData-Driven Deployment: The helm.yaml Configuration # This is where things get interesting. Instead of creating a separate role for each application (which led to a lot of code duplication), I\u0026rsquo;ve centralized all Helm application configurations into a single file: inventory/group_vars/all/helm.yaml. This file defines a helm_applications list, and each item describes one application to deploy.\nHere\u0026rsquo;s what a typical application definition looks like:\nhelm_applications: # cert-manager: Certificate management for Kubernetes - name: cert-manager chart: cert-manager version: v1.17.2 repo: https://charts.jetstack.io namespace: cert-manager create_namespace: true set_values: crds.enabled: \u0026#34;true\u0026#34; extraArgs[0]: \u0026#34;--dns01-recursive-nameservers-only\u0026#34; extraArgs[1]: \u0026#34;--dns01-recursive-nameservers=1.1.1.1:53\u0026#34; additional_manifests: - cert-manager-issuer # traefik: Ingress controller and load balancer - name: traefik chart: traefik repo: https://traefik.github.io/charts namespace: traefik create_namespace: true values_content: | service: type: LoadBalancer spec: loadBalancerIP: {{ vip_ingress_ip }} tlsStore: default: defaultCertificate: secretName: wildcard-tls additional_manifests: - traefik-wildcard-cert # longhorn: Distributed block storage - name: longhorn chart: longhorn version: v1.8.1 repo: https://charts.longhorn.io namespace: longhorn-system create_namespace: true ingress: enabled: true host: \u0026#34;longhorn.{{ ssl_local_domain }}\u0026#34; service_name: longhorn-frontend service_port: 80 additional_manifests: - longhorn-iscsi - longhorn-nfs Each application entry supports:\nBasic Helm config: name, chart, version, repo, namespace Helm values: via set_values (for simple key-value pairs) or values_content (for complex YAML blocks) Ingress routing: If ingress.enabled is true, an IngressRoute is automatically created Additional manifests: Custom resources like the ClusterIssuer for cert-manager or dependency jobs for Longhorn The beauty of this approach is that the deploy_helm_apps role simply loops through this list and applies templates. Here\u0026rsquo;s the simplified logic from roles/deploy_helm_apps/tasks/main.yaml:\n- name: Deploy helm charts ansible.builtin.template: src: templates/helm-chart.yaml.j2 dest: \u0026#34;/var/lib/rancher/rke2/server/manifests/{{ item.name }}-helm-chart.yaml\u0026#34; loop: \u0026#34;{{ helm_applications }}\u0026#34; - name: Deploy ingress routes for apps with ingress enabled ansible.builtin.template: src: templates/ingress-route.yaml.j2 dest: \u0026#34;/var/lib/rancher/rke2/server/manifests/{{ item.name }}-ingress-route.yaml\u0026#34; loop: \u0026#34;{{ helm_applications }}\u0026#34; when: - item.ingress is defined - item.ingress.enabled | default(false) - name: Deploy additional manifests ansible.builtin.template: src: \u0026#34;templates/additional/{{ manifest }}.yaml.j2\u0026#34; dest: \u0026#34;/var/lib/rancher/rke2/server/manifests/{{ manifest }}.yaml\u0026#34; loop: \u0026#34;{{ all_additional_manifests }}\u0026#34; This data-driven approach means:\nNo code duplication: One set of templates handles all applications Easy to maintain: Adding a new app is just adding a YAML block Consistent structure: Every app follows the same deployment pattern Clear overview: All deployed apps are visible in one config file Now let\u0026rsquo;s dive into how specific applications are deployed with this system.\nThe Playbook in Action: A Step-by-Step Breakdown # 1. Prepare All Nodes \u0026amp; Download RKE2 # This role prepares each node for RKE2 by configuring node-level prerequisites and placing the RKE2 binary on the system. It runs on all nodes (servers and agents) and performs the following tasks:\nDisable Swap: Ensures swap is disabled, a prerequisite for the Kubelet (source). Enable IP Forwarding: Enables IPv4 and IPv6 packet forwarding via sysctl, a networking requirement for all CNI plugins (source). Create Install Directory: Ensures the target directory for the binary (e.g., /usr/local/bin) exists. Download RKE2 Binary: Downloads the specific RKE2 version defined in your variables. Set Permissions: Makes the downloaded RKE2 binary executable. Create Server-Specific Directories: On servers only, it pre-creates the directories for RKE2 manifests (/var/lib/rancher/rke2/server/manifests for Helm Controller deployment) and configuration (/etc/rancher/rke2). 2. Bootstrapping the Control Plane (add-server) # This role targets only the servers group to configure and start the RKE2 server components, forming the control plane. This role includes:\nCreate Configuration Directory: Ensures /etc/rancher/rke2/ exists. Deploy Server Configuration: Deploys a config.yaml file from a template. This file dynamically populates key settings: token: The shared secret for the cluster. tls-san: A list of Subject Alternative Names for the API server\u0026rsquo;s TLS certificate, including the node\u0026rsquo;s IP, hostname, and the shared Virtual IP (vip). Conditional Logic: The role intelligently detects if it\u0026rsquo;s configuring the first server or a joining server. For joining servers, it adds the server URL to the config so the new node knows how to connect to the existing cluster. Enable and Start RKE2 Server: Enables and starts the rke2-server.service. Deploy CoreDNS Config (First Server Only): On the first server, it deploys a custom CoreDNS configuration. (Thanks to the Helm Controller, this will be automatically applied to the cluster). Create Kubeconfig: It waits for the server to be ready and then creates a .kube/config file in the remote user\u0026rsquo;s home directory. The main logic is in roles/add-server/tasks/main.yaml, and the Jinja template that injects the variables is at roles/add-server/templates/rke2-server-config.j2. You can see I\u0026rsquo;ve disabled certain things (like rke2-ingress-nginx) to make it more lightweight and replace them with my own (Traefik).\nwrite-kubeconfig-mode: \u0026#34;0644\u0026#34; token: {{ rke2_token }} # server: https://{{ hostvars[groups[\u0026#39;servers\u0026#39;][0]][\u0026#39;ansible_host\u0026#39;] }}:9345 // for join node tls-san: - {{ vip }} {% for host in groups[\u0026#39;servers\u0026#39;] %} - {{ hostvars[host][\u0026#39;ansible_host\u0026#39;] }} {% endfor %} node-label: - server=true disable-cloud-controller: true disable: - rke2-ingress-nginx kube-proxy-arg: - \u0026#34;proxy-mode=ipvs\u0026#34; - \u0026#34;ipvs-strict-arp=true\u0026#34; kube-proxy-extra-mount: - \u0026#34;/lib/modules:/lib/modules:ro\u0026#34; 3. Joining the Worker Nodes (add-agent) # This role targets all hosts in the agents and longhorn groups, configuring them as worker nodes and joining them to the cluster.\nCreate Configuration Directory: Ensures /etc/rancher/rke2/ exists. Create Systemd Service: Generates the rke2-agent.service systemd file from a template. Deploy Agent Configuration: Deploys a config.yaml file templated with the server URL (pointing to the first master) and the cluster token. It also includes specific settings for the longhorn group so that when we deploy Longhorn later, it knows which nodes it should run on. Enable and Start RKE2 Agent: Enables and starts the rke2-agent.service, which connects to the control plane. Here\u0026rsquo;s the agent config template. Notice in the node-label section, you can add a label for Longhorn nodes, this will be useful later when we deploy Longhorn.\nwrite-kubeconfig-mode: \u0026#34;0644\u0026#34; token: {{ rke2_token }} server: https://{{ hostvars[groups[\u0026#39;servers\u0026#39;][0]][\u0026#39;ansible_host\u0026#39;] }}:9345 node-label: - \u0026#34;agent=true\u0026#34; # - \u0026#34;node.longhorn.io/create-default-disk=true\u0026#34; kube-proxy-arg: - \u0026#34;proxy-mode=ipvs\u0026#34; - \u0026#34;ipvs-strict-arp=true\u0026#34; kube-proxy-extra-mount: - \u0026#34;/lib/modules:/lib/modules:ro\u0026#34; 4. High Availability with Kube-VIP # Image from https://www.sobyte.net/post/2021-09/use-kube-vip-ha-k8s-lb/\nThis role sets up a virtual IP (VIP) to create a highly available endpoint for the Kubernetes API server. kube-vip provides VIP and load balancing for bare-metal clusters. For control plane HA, it ensures continuous access to the API Server if a node fails. Additionally, it provides a native, in-cluster load balancer solution, giving us Service objects of type LoadBalancer without needing an external cloud provider.\nThis playbook performs the following tasks:\nDeploy RBAC Rules: Deploys a template to create the necessary ClusterRole and ClusterRoleBinding for Kube VIP.\nDeploy Kube VIP ConfigMap: Deploys a ConfigMap that configures the virtual IP address.\nDeploy Kube VIP Cloud Controller: Deploys a template that creates a DaemonSet (to run Kube VIP on every control-plane node) and the cloud controller (to provide LoadBalancer services).\nRemember when I mentioned the Helm Controller that RKE2 integrated? This makes deploying manifests easy. Any file found in /var/lib/rancher/rke2/server/manifests will automatically be deployed, similar to kubectl apply. However, this method isn\u0026rsquo;t always reliable, as it\u0026rsquo;s hard to debug and track what went wrong (as you\u0026rsquo;ll see in the next section). Therefore, I\u0026rsquo;ve chosen ArgoCD to deploy all my applications later.\n5. Ingress and SSL with Traefik \u0026amp; Cert-Manager # Image from https://msazure.club/using-traefik-ingress-with-cert-manager-and-aad-authentication/\nI didn\u0026rsquo;t want to access services via NodePort and port-forwarding. I figured I\u0026rsquo;d configure LoadBalancer services, since I already have kube-vip. However, I also don\u0026rsquo;t like remembering IPs for each service, so I thought I\u0026rsquo;d use an Ingress Controller.\nI need to be honest: this part was a nightmare, and I want to share the solutions so you don\u0026rsquo;t waste a day like I did.\nProblem 1 (Silent Fail): RKE2\u0026rsquo;s Helm controller silently failed because I forgot to rename my cert-manager-helm-chart.j2 file to .yaml.\nProblem 2 (Fetch Fail): When it finally loaded, the HelmChart couldn\u0026rsquo;t fetch the cert-manager repo. The logs were useless.\nSolution 2: I had to download the .tgz file, base64 encode it, and manually inject it into the chartContent field of the manifest just to see the real error.\nProblem 3 (DNS Fail): The real error was that DNS-01 validation was failing. My VM\u0026rsquo;s default DNS (/etc/resolv.conf) was pointing to my internal router, which couldn\u0026rsquo;t resolve the Let\u0026rsquo;s Encrypt validation domains.\nThe Fix: I had to go all the way back to Terraform and specify an external DNS server (like 1.1.1.1) in my cloud-init config. I\u0026rsquo;m still not 100% sure why this was the only fix, but it worked.\nEventually, I circled back to Traefik - mainly because I\u0026rsquo;m planning to integrate Authentik for authentication, and Traefik\u0026rsquo;s middlewares make that much easier.\nWith the new deploy_helm_apps approach, both Cert-Manager and Traefik are defined in helm.yaml. Here\u0026rsquo;s how they\u0026rsquo;re configured:\nCert-Manager configuration:\n- name: cert-manager chart: cert-manager version: v1.17.2 repo: https://charts.jetstack.io namespace: cert-manager create_namespace: true set_values: crds.enabled: \u0026#34;true\u0026#34; extraArgs[0]: \u0026#34;--dns01-recursive-nameservers-only\u0026#34; extraArgs[1]: \u0026#34;--dns01-recursive-nameservers=1.1.1.1:53\u0026#34; additional_manifests: - cert-manager-issuer # Deploys the ClusterIssuer Notice the extraArgs for DNS configuration - this is the fix for Problem 3 above. The additional_manifests field tells the role to also deploy the ClusterIssuer, which authenticates with Cloudflare:\n--- apiVersion: v1 kind: Secret metadata: name: cloudflare-api-token-secret namespace: cert-manager type: Opaque stringData: api-token: {{ ssl_cloudflare_api_token }} --- apiVersion: cert-manager.io/v1 kind: ClusterIssuer metadata: name: cloudflare-clusterissuer spec: acme: email: {{ ssl_email }} server: https://acme-v02.api.letsencrypt.org./directory privateKeySecretRef: name: cloudflare-key solvers: - dns01: cloudflare: apiTokenSecretRef: name: cloudflare-api-token-secret key: api-token After that, you need to configure Traefik to use this issuer to generate wildcard certificates. This means any service I create at *.dev.phuchoang.sbs will get SSL automatically. Here\u0026rsquo;s the Traefik configuration from helm.yaml:\nTraefik configuration:\n- name: traefik chart: traefik repo: https://traefik.github.io/charts namespace: traefik create_namespace: true values_content: | service: type: LoadBalancer spec: loadBalancerIP: {{ vip_ingress_ip }} tlsStore: default: defaultCertificate: secretName: wildcard-tls ingressRoute: dashboard: enabled: true matchRule: Host(`traefik.{{ ssl_local_domain }}`) entryPoints: [\u0026#34;websecure\u0026#34;] ports: web: redirections: entryPoint: to: websecure scheme: https permanent: true additional_manifests: - traefik-wildcard-cert # Deploys the Certificate resource The traefik-wildcard-cert manifest creates a Certificate that requests a wildcard cert from our ClusterIssuer:\napiVersion: cert-manager.io/v1 kind: Certificate metadata: name: wildcard-cert namespace: traefik spec: secretName: wildcard-tls dnsNames: - \u0026#34;{{ ssl_local_domain }}\u0026#34; - \u0026#34;*.{{ ssl_local_domain }}\u0026#34; issuerRef: name: cloudflare-clusterissuer kind: ClusterIssuer privateKey: rotationPolicy: Always additionalOutputFormats: - type: CombinedPEM 6. Deploying Core Services (Longhorn \u0026amp; ArgoCD) # Longhorn: For Persistent, Replicated Storage # Image from official docs https://longhorn.io/docs/1.10.1/concepts/\nLonghorn is a lightweight, distributed block storage system for Kubernetes. It provides persistent storage for stateful applications, replicates that storage for high availability, and can back up volumes to our TrueNAS NFS server.\nWe install Longhorn on the nodes we tagged with the node.longhorn.io/create-default-disk=true label back in the add-agent role. However, Longhorn has several prerequisites, including open-iscsi and an nfsv4 client. When I used the Longhorn CLI tool to check my nodes, I found I was missing them.\nLuckily, Longhorn offers manifest jobs to install these dependencies. With the new helm.yaml approach, we simply reference these in the additional_manifests field:\nLonghorn configuration:\n- name: longhorn chart: longhorn version: v1.8.1 repo: https://charts.longhorn.io namespace: longhorn-system create_namespace: true set_values: defaultSettings.createDefaultDiskLabeledNodes: \u0026#34;true\u0026#34; persistence.reclaimPolicy: \u0026#34;Retain\u0026#34; ingress: enabled: true host: \u0026#34;longhorn.{{ ssl_local_domain }}\u0026#34; service_name: longhorn-frontend service_port: 80 additional_manifests: - longhorn-iscsi # Installs iSCSI dependencies - longhorn-nfs # Installs NFS client dependencies Notice three things here:\nThe additional_manifests field automatically deploys the prerequisite jobs The ingress.enabled flag tells deploy_helm_apps to create an IngressRoute The set_values configures Longhorn to use our labeled nodes All of this happens automatically - the role loops through the configuration, deploys the manifests, creates the Helm chart, and sets up the ingress.\nArgoCD: Our GitOps Engine # Image from official docs https://www.cncf.io/blog/2020/12/17/solving-configuration-drift-using-gitops-with-argo-cd/\nArgoCD will play a prominent role in the cluster, as it will be responsible for deploying all of our applications in the next part of the series. With the centralized configuration, deploying ArgoCD is just another entry in helm.yaml:\nArgoCD configuration:\n- name: argo-cd chart: argo-cd version: v8.0.9 repo: https://argoproj.github.io/argo-helm namespace: argo-cd create_namespace: true values_content: | configs: params: server.insecure: true # We handle TLS at Traefik ingress: enabled: true host: \u0026#34;argo.{{ ssl_local_domain }}\u0026#34; service_name: argo-cd-argocd-server service_port: 80 The server.insecure: true setting is intentional - we\u0026rsquo;re terminating TLS at Traefik (our ingress controller), not at ArgoCD itself. This is a common pattern in Kubernetes where the ingress handles all TLS, and internal traffic is plain HTTP.\nThe power of this approach: If I want to add another application tomorrow (say, Prometheus or Grafana), I just add another block to helm.yaml. No new role, no new template files scattered around - just one YAML entry, and deploy_helm_apps handles the rest.\nThe Final Handoff: Getting kubectl Access # After all those tasks, we finally have a fully-fledged, working Kubernetes cluster. The last step is to get cluster access so we can control it with kubectl.\nNote: This section will be updated in a future post to cover OIDC authentication with HashiCorp Vault. Instead of fetching and storing kubeconfig files directly, we\u0026rsquo;ll integrate the cluster with Vault for centralized identity management and short-lived credentials. This provides better security through:\nZero-trust authentication via OIDC Role-based access control (RBAC) tied to Vault policies No long-lived credentials stored locally For now, you can fetch the kubeconfig from the first server node to get cluster access, but we\u0026rsquo;ll replace this with the Vault OIDC flow in the upcoming GitOps series.\nVerification # Once you have cluster access configured, you can verify the deployment. Here\u0026rsquo;s an example of checking the nodes:\ndev (default) in kubernetes-proxmox on master [!] is 📦 v0.1.0 via 🐍 v3.13.7 ❯ k get nodes NAME STATUS ROLES AGE VERSION dev-longhorn1 Ready \u0026lt;none\u0026gt; 2d3h v1.32.3+rke2r1 dev-longhorn2 Ready \u0026lt;none\u0026gt; 2d3h v1.32.3+rke2r1 dev-longhorn3 Ready \u0026lt;none\u0026gt; 2d3h v1.32.3+rke2r1 dev-server1 Ready control-plane,etcd,master 2d3h v1.32.3+rke2r1 dev-server2 Ready control-plane,etcd,master 2d3h v1.32.3+rke2r1 dev-server3 Ready control-plane,etcd,master 2d3h v1.32.3+rke2r1 You can also verify that all the Helm applications deployed successfully:\n❯ kubectl get helmcharts -A NAMESPACE NAME CHART REPO VERSION STATUS cert-manager cert-manager cert-manager https://charts.jetstack.io v1.17.2 Deployed traefik traefik traefik https://traefik.github.io/charts latest Deployed longhorn-sy.. longhorn longhorn https://charts.longhorn.io v1.8.1 Deployed argo-cd argo-cd argo-cd https://argoproj.github.io/argo-helm v8.0.9 Deployed After that, use kubectx and kubens (highly recommended tools) to switch your context and namespace. Then, verify the nodes.\ndev (default) in kubernetes-proxmox on  master [!] is 📦 v0.1.0 via 🐍 v3.13.7 ❯ k get nodes NAME STATUS ROLES AGE VERSION dev-longhorn1 Ready \u0026lt;none\u0026gt; 2d3h v1.32.3+rke2r1 dev-longhorn2 Ready \u0026lt;none\u0026gt; 2d3h v1.32.3+rke2r1 dev-longhorn3 Ready \u0026lt;none\u0026gt; 2d3h v1.32.3+rke2r1 dev-server1 Ready control-plane,etcd,master 2d3h v1.32.3+rke2r1 dev-server2 Ready control-plane,etcd,master 2d3h v1.32.3+rke2r1 dev-server3 Ready control-plane,etcd,master 2d3h v1.32.3+rke2r1 Summary \u0026amp; Next Steps # And just like that, we have a fully-fledged, highly-available Kubernetes cluster.\nThanks to Terraform, we can destroy and rebuild these 12 VMs in minutes. And thanks to Ansible, we can configure them all from scratch with a single command (ansible-playbook site.yaml).\nWe now have:\nA multi-master RKE2 control plane A Virtual IP for HA with kube-vip An ingress controller (Traefik) Automatic SSL certificates (cert-manager) Distributed persistent storage (Longhorn) The cluster is built. The platform is ready. In the next and final part of this series, we\u0026rsquo;ll do the fun stuff: use ArgoCD to deploy all of our applications with a true GitOps workflow.\n","date":"11 November 2025","externalUrl":null,"permalink":"/posts/on-premise-provison-ansible/","section":"Posts","summary":"People are often scared to learn Kubernetes because it’s difficult to set up an environment to play around with. Sure, you can use a project like minikube to learn on your own, but running it on your laptop seems limited, and you may not get to experience all the awesome features Kubernetes offers (remember, it was born for high availability). Or, if you have money, you can use a managed Kubernetes cluster from one of the cloud providers. Often, they’ve abstracted away so many things under the hood that you only need to learn kubectl and you’re good to go.\n","title":"On-Premise 101 (Part 5): From Terraform VMs to a K8s Cluster with Ansible","type":"posts"},{"content":"","date":"6 November 2025","externalUrl":null,"permalink":"/tags/devops/monitoring/grafana/","section":"Tags","summary":"","title":"devops/monitoring/grafana","type":"tags"},{"content":"","date":"6 November 2025","externalUrl":null,"permalink":"/tags/devops/monitoring/prometheus/","section":"Tags","summary":"","title":"devops/monitoring/prometheus","type":"tags"},{"content":"I\u0026rsquo;m the kind of person who will happily spend 10 hours building an automation script just to save 10 minutes of manual work every day. If that sounds like you, you\u0026rsquo;re in the right place.\nAfter covering the foundations of our hypervisor and storage platform, it\u0026rsquo;s time to move on to the most interesting part of this series: provisioning a Kubernetes cluster on Proxmox. This means creating a bunch of VMs, and I refuse to do that manually. Luckily, I have learned Terraform and Ansible somewhat enough to be able to create this project below to automate it all.\nphuchoang2603/kubernetes-proxmox auto provision k8s to proxmox using terraform + ansible + github actions + vault HCL 28 3 This post is first part of my deep dive on the project, focusing on how I use Terraform to create a repeatable, automated, and scalable VMs cluster on Proxmox, all with a single command. If you want to see how this project performs in action, you can watch the video below.\nWhy Automate? The Case for IaC with Terraform # \u0026ldquo;But what is Terraform?\u0026rdquo; you might ask. Again, we need to consider the core problem it solves to truly appreciate it.\nBefore Terraform, and before the \u0026ldquo;cloud\u0026rdquo; even existed, people deployed infrastructure by manually creating VMs with repeated configurations (CPU, RAM, network, firewall) and then SSH into each machine to run commands,or if they were fancy, a bash script. This did the job, but doing it manually was error-prone, time-consuming, and a nightmare to document.\nThe \u0026ldquo;cloud\u0026rdquo; was born to help solve this: it\u0026rsquo;s still someone\u0026rsquo;s computer, but those computers are now abstracted with many services on top, so we no longer need to build the infrastructure ourselves and can create VMs faster. Heck, we might not need to create VMs; instead, we just need to write a Dockerfile, or even just the function of the code, to get it deployed directly. However, we still often need to manually log in to a website and create VMs through a clunky UI.\nThat\u0026rsquo;s where Terraform comes in. It\u0026rsquo;s an open-source, Infrastructure as Code (IaC) tool from HashiCorp that lets you define and provision infrastructure using a declarative configuration language called HCL. It allows you to build, change, and manage infrastructure safely and efficiently. By writing code to describe your desired end state, Terraform automates the process of creating, updating, and destroying infrastructure, reducing manual effort and improving consistency.\nBecause your infrastructure is now code, it inherits awesome features like version history (via Git) and a documented process. If something goes wrong, you can tear down and bring up the entire infrastructure again with just two commands: terraform destroy and terraform apply. If someone new joins the team, they can easily get a grasp of the infrastructure by reading the code, without endless clicking on a cloud provider\u0026rsquo;s website.\nBy using Terraform, I can easily create tons of VMs on Proxmox and show you exactly what I\u0026rsquo;m doing.\nMy Terraform Blueprint: Project Structure \u0026amp; Principles # In the image above, we see 4 files on the left. These also represent the 4 primary components of a Terraform project: providers, variables, state, and instances (resources/modules).\nBefore diving into each component, let\u0026rsquo;s establish the core design principles that keep our Terraform code from becoming a messy nightmare (otherwise, it\u0026rsquo;s no better than the clunky UI):\nSeparation of Concerns: Each file should has a distinct and clear purpose. Variable-Driven Configuration: The infrastructure\u0026rsquo;s composition (how many nodes, their IPs, etc.) is defined in variables file, not hardcoded in the logic. Modularity: Reusable components (like a virtual machine) are encapsulated in modules. Environment Isolation: Configurations for different environments (like dev and prod) are kept separate. To give you a map of what I\u0026rsquo;m doing, here\u0026rsquo;s my Terraform file structure:\n. ├── create_user_config.tf ├── download_img.tf ├── env │ ├── dev │ │ ├── backend.hcl │ │ ├── k8s_nodes.json │ │ ├── longhorn_nodes.json │ │ └── main.tfvars │ └── prod │ ├── backend.hcl │ ├── k8s_nodes.json │ ├── longhorn_nodes.json │ └── main.tfvars ├── modules │ └── vm │ ├── main.tf │ ├── outputs.tf │ └── variables.tf ├── provider.tf ├── variables.tf └── vms.tf Connecting to Proxmox and S3 # provider.tf: Defining Our Connections # Providers are plugins that enable Terraform to interact with various platforms (AWS, Azure, GCP) and services (Kubernetes, DNS, etc.). They understand the resource types and make the API calls to create, read, update, and delete them.\nIn my case, I need to spin up VMs on Proxmox, so I looked for a provider that integrates with it. There are two prominent ones: Telmate/terraform-provider-proxmox and bpg/terraform-provider-proxmox. The one from telmate has been around for a long time but is limited in functionality. The one from bpg is newer and allows you to manage every aspect of a Proxmox environment. Thus, I opted for bpg.\nLater, I\u0026rsquo;ll introduce the concept of \u0026ldquo;remote state,\u0026rdquo; which uses a backend provider. I\u0026rsquo;ll be using an S3-compatible object storage (Minio), so I\u0026rsquo;ll define that as well. Enough talk, here\u0026rsquo;s my provider.tf file.\nterraform { required_version = \u0026#34;\u0026gt;= 1.6.6\u0026#34; required_providers { `Proxmox` = { source = \u0026#34;bpg/proxmox\u0026#34; version = \u0026#34;0.77.1\u0026#34; } } backend \u0026#34;s3\u0026#34; {} } provider \u0026#34;proxmox\u0026#34; { endpoint = var.proxmox_endpoint insecure = var.proxmox_insecure min_tls = var.proxmox_min_tls username = var.proxmox_username password = var.proxmox_password } Notice the version constraint - this is a critical best practice that locks your project to a specific provider version, preventing unexpected breaking changes from automatic updates.\nterraform.tfstate: Using a Remote Backend # Terraform state is a crucial component that stores a mapping of your configured resources to the real-world objects that have been created. It’s how Terraform knows what it manages.\nThere are two primary ways to store state: local (a terraform.tfstate file on your computer) and remote (on an online storage service like S3). The local state is fine for solo experiments but is disastrous for teams where multiple people are trying to make changes. What happens is Terraform can\u0026rsquo;t match the desired state (your code) with the actual state (on the server) because there\u0026rsquo;s no single source of truth.\nTherefore, I decided to use a remote backend. The credentials for it are handled by the env/dev/backend.hcl file below.\naccess_key = \u0026#34;terraform\u0026#34; secret_key = \u0026#34;QmwRghG9X80kC%\u0026#34; bucket = \u0026#34;terraform\u0026#34; key = \u0026#34;dev.tfstate\u0026#34; region = \u0026#34;us-east-1\u0026#34; # --- MinIO Specific Settings --- # The URL of your MinIO server endpoints = { s3 = \u0026#34;http://10.69.1.102:9000\u0026#34; } # Use path-style addressing (e.g., http://minio/bucket/key) use_path_style = true # Skip AWS-specific validation checks skip_credentials_validation = true # Skip AWS related checks and validations skip_requesting_account_id = true skip_metadata_api_check = true skip_region_validation = true What\u0026rsquo;s more, using remote state, I can now use my Terraform code for multiple backends. I just need to keep my two backends (dev and prod) separate and I can decide which environment to initialize Terraform with by running:\nterraform init -reconfigure -backend-config=env/dev/backend.hcl # For production: # terraform init -reconfigure -backend-config=env/prod/backend.hcl This keeps your core configuration clean and environment-agnostic. It securely stores your state in a remote, centralized location, enabling collaboration and preventing state file loss.\nbackend.hcl: Using Minio for Remote State # To get the credentials for the backend.hcl file, you first need to install Minio on your storage platform (in my case, TrueNAS). You may also need to install minio-console. This is because we need to create an IAM user for terraform and get its credentials, and the primary minio container UI has removed some of those features. The minio-console brings those features back. The \u0026ldquo;Inputs\u0026rdquo;: Variables, tfvars, and JSON # Next up are variables, which are the parameters for your Terraform configuration. They allow you to create flexible and reusable code by separating the \u0026ldquo;what\u0026rdquo; (the resource logic) from the \u0026ldquo;how\u0026rdquo; (the specific values).\nvariables.tf: Defining the \u0026ldquo;API\u0026rdquo; # This file acts as the \u0026ldquo;API\u0026rdquo; for the entire project. It defines every possible input, from provider credentials to VM specs, and sets optional default values. This file makes the project self-documenting.\nvariable \u0026#34;env\u0026#34; { description = \u0026#34;Environment name (e.g., dev, prod)\u0026#34; type = string } variable \u0026#34;proxmox_endpoint\u0026#34; { type = string description = \u0026#34;Proxmox API endpoint (e.g., https://your-proxmox-ip:8006)\u0026#34; } variable \u0026#34;proxmox_insecure\u0026#34; { type = bool description = \u0026#34;Skip TLS verification\u0026#34; default = true } variable \u0026#34;proxmox_min_tls\u0026#34; { type = string description = \u0026#34;Minimum TLS version\u0026#34; default = \u0026#34;1.3\u0026#34; } variable \u0026#34;proxmox_username\u0026#34; { description = \u0026#34;Proxmox username\u0026#34; type = string } variable \u0026#34;proxmox_password\u0026#34; { description = \u0026#34;Proxmox password\u0026#34; type = string sensitive = true } variable \u0026#34;proxmox_ssh_public_key\u0026#34; { description = \u0026#34;SSH public key for VM access\u0026#34; type = string } variable \u0026#34;vm_node_name\u0026#34; { description = \u0026#34;Proxmox node where VMs are created\u0026#34; type = string } variable \u0026#34;vm_datastore_id\u0026#34; { description = \u0026#34;Proxmox datastore ID where snippets cloud img are stored\u0026#34; type = string } variable \u0026#34;vm_bridge\u0026#34; { description = \u0026#34;Network bridge used for VM network\u0026#34; type = string } variable \u0026#34;vm_timezone\u0026#34; { description = \u0026#34;Timezone for the VM\u0026#34; type = string } variable \u0026#34;vm_username\u0026#34; { description = \u0026#34;Username for the VM template\u0026#34; type = string } variable \u0026#34;vm_ip_gateway\u0026#34; { description = \u0026#34;Gateway for Kubernetes VMs\u0026#34; type = string } variable \u0026#34;dns_server\u0026#34; { description = \u0026#34;DNS server for Kubernetes VMs\u0026#34; type = string } variable \u0026#34;k8s_cpu_cores\u0026#34; { description = \u0026#34;Number of CPU cores per VM\u0026#34; type = number } variable \u0026#34;k8s_cpu_type\u0026#34; { description = \u0026#34;CPU type for VM\u0026#34; type = string } variable \u0026#34;k8s_memory_mb\u0026#34; { description = \u0026#34;Memory size in MB per VM\u0026#34; type = number } variable \u0026#34;k8s_datastore_id\u0026#34; { description = \u0026#34;k8s datastore ID where VM disks are stored\u0026#34; type = string } variable \u0026#34;k8s_disk_size_gb\u0026#34; { description = \u0026#34;Disk size in GB for VM disk\u0026#34; type = number } variable \u0026#34;longhorn_cpu_cores\u0026#34; { description = \u0026#34;Number of CPU cores per VM\u0026#34; type = number } variable \u0026#34;longhorn_cpu_type\u0026#34; { description = \u0026#34;CPU type for VM\u0026#34; type = string } variable \u0026#34;longhorn_memory_mb\u0026#34; { description = \u0026#34;Memory size in MB per VM\u0026#34; type = number } variable \u0026#34;longhorn_datastore_id\u0026#34; { description = \u0026#34;longhorn datastore ID where VM disks are stored\u0026#34; type = string } variable \u0026#34;longhorn_disk_size_gb\u0026#34; { description = \u0026#34;Disk size in GB for VM disk\u0026#34; type = number } locals { k8s_nodes = jsondecode(file(\u0026#34;${path.root}/env/${var.env}/k8s_nodes.json\u0026#34;)) longhorn_nodes = jsondecode(file(\u0026#34;${path.root}/env/${var.env}/longhorn_nodes.json\u0026#34;)) } main.tfvars: Setting Environment Values # While variables.tf defines the inputs, this file provides the concrete values for a specific environment. The reason I separate my *.tfvars files into env/dev/ and env/prod/ is the same as for the backend state: it allows me to spin up a completely new environment just by creating a new directory. You can tell Terraform which one to use like this:\nterraform apply -var-file=\u0026#34;env/dev/main.tfvars\u0026#34; # For production: # terraform apply -var-file=\u0026#34;env/prod/main.tfvars\u0026#34; Here\u0026rsquo;s an example of env/dev/main.tfvars. The variable names here must exactly match the names in variables.tf.\n# Environment name env = \u0026#34;dev\u0026#34; # `Proxmox` API details - PLEASE FILL THESE IN proxmox_endpoint = \u0026#34;https://10.69.1.1:8006/\u0026#34; proxmox_username = \u0026#34;root@pam\u0026#34; proxmox_password = \u0026#34;Phuc@2006\u0026#34; proxmox_ssh_public_key = \u0026#34;/home/felix/.ssh/id_ed25519.pub\u0026#34; # Absolute path of your ssh public key on your machine # General VM settings - PLEASE REVIEW AND ADJUST vm_node_name = \u0026#34;pve\u0026#34; # `Proxmox` node where VMs are created vm_datastore_id = \u0026#34;truenas\u0026#34; # storage for downloading cloud img, storing snippets, etc. vm_bridge = \u0026#34;vmbr0\u0026#34; vm_timezone = \u0026#34;America/New_York\u0026#34; vm_username = \u0026#34;ubuntu\u0026#34; vm_ip_gateway = \u0026#34;10.69.0.1\u0026#34; dns_server = \u0026#34;1.1.1.1\u0026#34; # k8s cluster settings k8s_cpu_cores = 2 k8s_cpu_type = \u0026#34;x86-64-v2-AES\u0026#34; k8s_memory_mb = 4096 k8s_disk_size_gb = 64 k8s_datastore_id = \u0026#34;local-lvm\u0026#34; # longhorn cluster settings longhorn_cpu_cores = 2 longhorn_cpu_type = \u0026#34;x86-64-v2-AES\u0026#34; longhorn_memory_mb = 2048 longhorn_disk_size_gb = 300 longhorn_datastore_id = \u0026#34;local-lvm\u0026#34; k8s_nodes.json: A Dynamic Node Inventory # locals { k8s_nodes = jsondecode(file(\u0026#34;${path.root}/env/${var.env}/k8s_nodes.json\u0026#34;)) longhorn_nodes = jsondecode(file(\u0026#34;${path.root}/env/${var.env}/longhorn_nodes.json\u0026#34;)) } You might have noticed the locals block at the end of variables.tf. This block dynamically loads the correct node inventory based on the env variable and makes it available to the rest of your configuration files. For example, it will read env/dev/k8s_nodes.json, which is the source of truth for my Kubernetes nodes. It defines each node’s name, ID, and IP address. To scale the cluster, you simply add a new entry to it.\n{ \u0026#34;dev-server1\u0026#34;: { \u0026#34;vm_id\u0026#34;: 111, \u0026#34;node\u0026#34;: \u0026#34;pve\u0026#34;, \u0026#34;role\u0026#34;: \u0026#34;servers\u0026#34;, \u0026#34;address\u0026#34;: \u0026#34;10.69.1.111/16\u0026#34; }, \u0026#34;dev-server2\u0026#34;: { \u0026#34;vm_id\u0026#34;: 112, \u0026#34;node\u0026#34;: \u0026#34;pve2\u0026#34;, \u0026#34;role\u0026#34;: \u0026#34;servers\u0026#34;, \u0026#34;address\u0026#34;: \u0026#34;10.69.1.112/16\u0026#34; }, \u0026#34;dev-server3\u0026#34;: { \u0026#34;vm_id\u0026#34;: 113, \u0026#34;node\u0026#34;: \u0026#34;pve3\u0026#34;, \u0026#34;role\u0026#34;: \u0026#34;servers\u0026#34;, \u0026#34;address\u0026#34;: \u0026#34;10.69.1.113/16\u0026#34; } } Not only does this make it easy to define multiple nodes, but it also helps Ansible (in the next part) read this same file to configure the IPs.\nThe \u0026ldquo;Blueprint\u0026rdquo;: Assembling the Cluster # Now that we have our variables configured, Terraform can talk to our infrastructure. The next step is to define what to say.\nThese are the .tf files that contain the logic - the blueprint for your infrastructure. They define resources, call modules, and orchestrate the entire deployment.\ndownload_img.tf: Acquiring the Base Image # resource \u0026#34;proxmox_virtual_environment_download_file\u0026#34; \u0026#34;ubuntu_cloud_image\u0026#34; { content_type = \u0026#34;iso\u0026#34; datastore_id = var.vm_datastore_id node_name = var.vm_node_name file_name = \u0026#34;${var.env}-noble-server-cloudimg-amd64.img\u0026#34; url = \u0026#34;https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img\u0026#34; } This file has one job: download the Ubuntu cloud image from the internet and store it in Proxmox. The proxmox_virtual_environment_download_file resource from the bpg/proxmox provider handles this, ensuring the base image is available before any VMs are created. Its output ID is used later to specify the base disk for the VMs installation.\nThis ISO file is special because you can modify the VM configuration directly with the cloud-init bundle with it.\nResult: create_user_config.tf: Preparing Cloud-Init # data \u0026#34;local_file\u0026#34; \u0026#34;ssh_public_key\u0026#34; { filename = var.proxmox_ssh_public_key } resource \u0026#34;proxmox_virtual_environment_file\u0026#34; \u0026#34;user_data_cloud_config\u0026#34; { content_type = \u0026#34;snippets\u0026#34; datastore_id = var.vm_datastore_id node_name = var.vm_node_name source_raw { data = \u0026lt;\u0026lt;-EOF #cloud-config host-name: ubuntu_cloud_image timezone: ${var.vm_timezone} users: - default - name: ${var.vm_username} groups: - sudo shell: /bin/bash ssh_authorized_keys: - ${trimspace(data.local_file.ssh_public_key.content)} sudo: ALL=(ALL) NOPASSWD:ALL package_update: true packages: - qemu-guest-agent - net-tools - curl - cryptsetup runcmd: - systemctl start qemu-guest-agent EOF file_name = \u0026#34;user-data-cloud-config.yaml\u0026#34; } } resource \u0026#34;proxmox_virtual_environment_file\u0026#34; \u0026#34;meta_data_cloud_config\u0026#34; { for_each = merge(local.k8s_nodes, local.longhorn_nodes) content_type = \u0026#34;snippets\u0026#34; datastore_id = var.vm_datastore_id node_name = var.vm_node_name source_raw { data = \u0026lt;\u0026lt;-EOF #cloud-config local-hostname: ${each.key} EOF file_name = \u0026#34;${each.key}-meta-data-cloud-config.yaml\u0026#34; } } This is one of the most important files for automation. It creates the configuration “snippets” that cloud-init will use to configure the VMs on their first boot.\nuser_data_cloud_config: This is a generic configuration applied to all VMs. It sets the timezone, creates a default user (ubuntu), injects your SSH public key for passwordless access, and installs essential packages like qemu-guest-agent. meta_data_cloud_config: This creates a specific metadata file for each VM. It iterates through the node maps loaded in locals.tf and creates a snippet that sets the correct hostname for each machine (e.g., dev-server1). Result: modules/vm/: The Reusable VM \u0026ldquo;Factory\u0026rdquo; # This directory encapsulates all the logic for creating a single Proxmox VM. It\u0026rsquo;s a Terraform module—a reusable package of configuration, much like a \u0026ldquo;class\u0026rdquo; in object-oriented programming.\nvariables.tf: This is the “order form” for the factory. It defines all the parameters needed to create one VM, such as its name, CPU cores, memory, IP address, etc. main.tf: This is the “assembly line.” It contains the proxmox_virtual_environment_vm resource block. It takes all the variables and uses them to configure the VM’s properties, from its CPU and memory to its disk and network settings. outputs.tf: This defines the “shipping label,” outputting the final vm_id of the created machine. # main.tf terraform { required_providers { `Proxmox` = { source = \u0026#34;bpg/proxmox\u0026#34; version = \u0026#34;0.77.1\u0026#34; } } } resource \u0026#34;proxmox_virtual_environment_vm\u0026#34; \u0026#34;vm\u0026#34; { name = var.name node_name = var.node_name vm_id = var.vm_id agent { enabled = true } stop_on_destroy = true machine = \u0026#34;q35\u0026#34; bios = \u0026#34;ovmf\u0026#34; description = \u0026#34;Cloud-Init ready Kubernetes template managed by Terraform\u0026#34; cpu { cores = var.cpu_cores type = var.cpu_type } memory { dedicated = var.memory_mb } efi_disk { datastore_id = var.datastore_id type = \u0026#34;4m\u0026#34; } disk { datastore_id = var.datastore_id file_id = var.disk_file_id interface = \u0026#34;virtio0\u0026#34; iothread = true discard = \u0026#34;on\u0026#34; size = var.disk_size_gb ssd = true } network_device { bridge = var.bridge } initialization { dns { servers = [var.dns_server] } ip_config { ipv4 { address = var.ip_address gateway = var.ip_gateway } } user_data_file_id = var.user_data_file_id meta_data_file_id = var.meta_data_file_id } } vms.tf: Orchestrating the Build with for_each # Now for the magic. All that setup—the variables, the JSON file, the module—was for this. This one file is our master orchestration file. Watch how we use for_each to loop through our JSON node list and pass the values to our VM module. This is how we create 12 (or 100) VMs as easily as one.\nmodule \u0026#34;k8s_nodes\u0026#34; { source = \u0026#34;./modules/vm\u0026#34; for_each = local.k8s_nodes name = each.key node_name = each.value.node vm_id = each.value.vm_id cpu_cores = var.k8s_cpu_cores cpu_type = var.k8s_cpu_type memory_mb = var.k8s_memory_mb datastore_id = var.k8s_datastore_id disk_file_id = proxmox_virtual_environment_download_file.ubuntu_cloud_image.id disk_size_gb = var.k8s_disk_size_gb bridge = var.vm_bridge dns_server = var.dns_server ip_address = each.value.address ip_gateway = var.vm_ip_gateway user_data_file_id = proxmox_virtual_environment_file.user_data_cloud_config.id meta_data_file_id = proxmox_virtual_environment_file.meta_data_cloud_config[each.key].id } module \u0026#34;longhorn_nodes\u0026#34; { source = \u0026#34;./modules/vm\u0026#34; for_each = local.longhorn_nodes name = each.key node_name = each.value.node vm_id = each.value.vm_id cpu_cores = var.longhorn_cpu_cores cpu_type = var.longhorn_cpu_type memory_mb = var.longhorn_memory_mb datastore_id = var.longhorn_datastore_id disk_file_id = proxmox_virtual_environment_download_file.ubuntu_cloud_image.id disk_size_gb = var.longhorn_disk_size_gb bridge = var.vm_bridge dns_server = var.dns_server ip_address = each.value.address ip_gateway = var.vm_ip_gateway user_data_file_id = proxmox_virtual_environment_file.user_data_cloud_config.id meta_data_file_id = proxmox_virtual_environment_file.meta_data_cloud_config[each.key].id } For every entry in local.k8s_nodes (and longhorn_nodes) variables, this file calls the ./modules/vm module, passing in the specific configuration. It combines:\nThe node-specific data from the JSON file (each.key, each.value.address). The generic VM settings from main.tfvars (var.k8s_cpu_cores). The IDs of the resources created in other files (the downloaded image and the cloud-init snippets). Result: Conclusion: The 12-Hour, 12-Minute Payoff # And with that, I\u0026rsquo;ve successfully created 12 VMs on Proxmox, all with specific, defined configurations, just by running terraform apply.\nYou could say I took 12 hours to automate a 12-minute job, and you\u0026rsquo;d be right. But I\u0026rsquo;m still incredibly proud of this work. Now, I can spin down the entire cluster and spin it back up again, at will, in minutes. What\u0026rsquo;s more, I can scale up or change the configuration for all my VMs just by editing a single JSON file.\nThat is the power of well-designed Infrastructure as Code.\n","date":"6 November 2025","externalUrl":null,"permalink":"/posts/on-premise-provision-terraform/","section":"Posts","summary":"I’m the kind of person who will happily spend 10 hours building an automation script just to save 10 minutes of manual work every day. If that sounds like you, you’re in the right place.\n","title":"On-Premise 101 (Part 4): Automating 12 VMs creation on Proxmox with Terraform","type":"posts"},{"content":"","date":"6 November 2025","externalUrl":null,"permalink":"/tags/programming/lang/python/","section":"Tags","summary":"","title":"programming/lang/python","type":"tags"},{"content":"","date":"6 November 2025","externalUrl":"https://github.com/phuchoang2603/realtime-credit-card-fraud-detection","permalink":"/projects/realtime-credit-card-fraud-detection/","section":"Projects","summary":"This repository contains a complete end-to-end system for real-time credit card fraud detection, including data analysis notebooks, a machine learning API, and infrastructure-as-code for cloud deployment. The system is designed with a modern, observable, and scalable architecture.","title":"Scalable Real-Time Credit Card Fraud Detection System","type":"projects"},{"content":"This is my story of how I attempt to build an AI-powered pipeline for ArXiv papers. It was a journey that started with a cool idea about AI agents and ended with me wrestling Docker, n8n, and Python into submission.\nphuchoang2603/sis-arxiv-vad-papers auto scrape and host arxiv papers on the topic \u0026ldquo;video anomaly detection\u0026rdquo; Python 2 0 Part 1: The AI Agent Dream (and Subsequent Nightmare) # My first idea was to build a smart AI agent using n8n that could talk to ArXiv. I wanted an LLM that could actually search for papers, download them, and even read them.\nThe Docker Model Context Protocol (MCP) looked like the perfect tool for this. I found an arxiv-mcp-server on GitHub and my first task was just getting the thing to build.\nBuilding the ArXiv Server # blazickjp/arxiv-mcp-server A local MCP server for agent literature work. Original-LaTeX section reads, BibTeX from arXiv metadata, and topic watches. Papers stay on disk. Search is optional. Python 3211 257 I found the repo above, which had already configured an mcp server just as I wanted. However, when I attempted to get it running, I found the original Dockerfile was\u0026hellip; a bit much. I figured I could simplify it using uv, since its official base images are clean and come with it pre-installed.\nMy new Dockerfile was way simpler:\n# Use a single Python base image with \u0026#39;uv\u0026#39; pre-installed FROM ghcr.io/astral-sh/uv:python3.11-bookworm-slim WORKDIR /app COPY . . # Install the project and all its deps RUN uv pip install . --system # Run the server ENTRYPOINT [\u0026#34;python\u0026#34;, \u0026#34;-m\u0026#34;, \u0026#34;arxiv_mcp_server\u0026#34;] The \u0026ldquo;Stateless\u0026rdquo; Problem # With the server built, I set up a custom catalog.yaml above to define my tools and hooked it all into docker-compose.yml. A custom catalog is essentially your own personal, self-contained list of MCP servers. Instead of relying on a public registry, you define everything about the servers you want to use in a single catalog.yaml file.\nThe process is straightforward:\nCreate catalog.yaml: You define one or more servers in a YAML file following the specified format. Mount the Catalog: In your docker-compose.yml file, you use a volume mount to make your local catalog.yaml file available inside the gateway container. volumes: - ./catalog.yaml:/mcp/catalog.yaml Tell the Gateway to Use It: You use command arguments to point the gateway to your mounted catalog file and specify which server(s) from that file you want to activate. command: - --catalog=/mcp/catalog.yaml - --servers=duckduckgo registry: arxiv-mcp-server: title: \u0026#34;ArXiv MCP Server\u0026#34; description: \u0026#34;An MCP server that enables AI assistants to search, download, and read papers from the arXiv research repository.\u0026#34; type: \u0026#34;server\u0026#34; image: \u0026#34;arxiv-mcp-server:latest\u0026#34; tools: - name: \u0026#34;search_papers\u0026#34; - name: \u0026#34;download_paper\u0026#34; - name: \u0026#34;list_papers\u0026#34; - name: \u0026#34;read_paper\u0026#34; - name: \u0026#34;deep-paper-analysis\u0026#34; env: - name: \u0026#34;ARXIV_STORAGE_PATH\u0026#34; value: \u0026#34;/data\u0026#34; volumes: - \u0026#34;/mnt/storage/media/docs:/data\u0026#34; docker-compose file\nservices: n8n: image: docker.n8n.io/n8nio/n8n hostname: n8n ports: - \u0026#34;5678:5678\u0026#34; environment: - N8N_ENFORCE_SETTINGS_FILE_PERMISSIONS=true - N8N_HOST=${SUBDOMAIN}.${DOMAIN_NAME} - N8N_PORT=5678 - N8N_PROTOCOL=http - N8N_RUNNERS_ENABLED=true - WEBHOOK_URL=https://${SUBDOMAIN}.${DOMAIN_NAME}/ - OLLAMA_HOST=${OLLAMA_HOST:-ollama:11434} - TZ=${GENERIC_TIMEZONE} volumes: - ${APPDATA}/n8n/storage:/home/node/.n8n - ${SHARED_FOLDER}:/files restart: always mcp-gateway: image: docker/mcp-gateway hostname: mcp-gateway ports: - \u0026#34;8811:8811\u0026#34; command: - --servers=arxiv-mcp-server - --catalog=/mcp/catalog.yaml - --transport=sse - --port=8811 volumes: - /var/run/docker.sock:/var/run/docker.sock - ./catalog.yaml:/mcp/catalog.yaml I then wired it up in n8n with an AI Agent node, first trying a local qwen3 model (way too slow) and then switching to gpt-4o-mini (much faster).\nIt worked! The agent called the search_papers tool.\nThen, I hit the first major wall. The agent would download_paper, and on the very next step, the read_paper tool would fail.\nThe Problem: The MCP gateway, by default, is stateless. It spins up a brand new container for every single tool call. The container that downloaded the paper was instantly destroyed, so the new container for read_paper had no idea the file existed. Bruh.\nThe \u0026ldquo;Static\u0026rdquo; Mode Saga # The fix was static=true mode. This tells the gateway to connect to an already-running server container. I dutifully refactored my docker-compose.yml to have mcp-gateway depend on a long-running mcp-arxiv-server.\nmcp-gateway: image: docker/mcp-gateway hostname: mcp-gateway ports: - \u0026#34;8811:8811\u0026#34; command: - --servers=arxiv-mcp-server,duckduckgo - --static=true - --transport=streaming - --port=8811 depends_on: - mcp-arxiv-server mcp-arxiv-server: image: mcp/arxiv-mcp-server entrypoint: [\u0026#34;/docker-mcp/misc/docker-mcp-bridge\u0026#34;, \u0026#34;python\u0026#34;, \u0026#34;-m\u0026#34;, \u0026#34;arxiv_mcp_server\u0026#34;] init: true labels: - docker-mcp=true - docker-mcp-tool-type=mcp - docker-mcp-name=arxiv-mcp-server - docker-mcp-transport=stdio volumes: - type: image source: docker/mcp-gateway target: /docker-mcp - ${SHARED_FOLDER}:/app/papers It failed.\nI tried the exact same setup with the official duckduckgo server, and it worked perfectly. My ArXiv server? Nothing. Just network errors.\nI was losing my mind, so I filed a GitHub issue. A collaborator saved me. Turns out, the gateway auto-prefixes the server name with mcp- to find the service.\nMy service was named mcp-arxiv-server. The gateway was looking for mcp-mcp-arxiv-server.\nI renamed my service to arxiv-mcp-server (so the gateway would find it at mcp-arxiv-mcp-server) and just like that, it worked. The agent could finally search, download, and read papers in one session.\nPart 2: The New Task - Batch Processing 300 PDFs # Right after that win, I had a meeting with Lokman Belkit. We shifted gears. I now had a folder of 300 PDFs he\u0026rsquo;d sent me. The priority was no longer the live agent, but batch-processing this existing data.\nI needed a good PDF-to-Markdown converter. LlamaParse looked amazing, but you have to negotiate a license. No thanks. I settled on Docling because it was open-source and had a docling-serve image.\nConfiguring docling-serve # This took way longer than I expected. docling-serve is great, but it needs to download all its models before it can run. I ended up creating a two-stage setup in my docker-compose.yml:\ndocling-serve-initial: A service that runs once (restart: \u0026quot;no\u0026quot;) and just runs the download command, saving the models to a shared Docker volume. docling-serve: The main server. It depends_on the initial service, mounts that same volume, and reads the pre-downloaded models. services: # To download the required models before first run docling-serve-initial: image: ghcr.io/docling-project/docling-serve-cu126:main command: - docling-tools - models - download - --all volumes: - ${APPDATA}/n8n/docling_artifacts:/opt/app-root/src/.cache/docling/models restart: \u0026#34;no\u0026#34; # For document parsing docling-serve: image: ghcr.io/docling-project/docling-serve-cu126:main hostname: docling-serve ports: - \u0026#34;5001:5001\u0026#34; environment: DOCLING_SERVE_ENABLE_UI: \u0026#34;true\u0026#34; NVIDIA_VISIBLE_DEVICES: \u0026#34;all\u0026#34; DOCLING_SERVE_ARTIFACTS_PATH: \u0026#34;/models\u0026#34; DOCLING_SERVE_ENABLE_REMOTE_SERVICES: \u0026#34;true\u0026#34; DOCLING_SERVE_ALLOW_EXTERNAL_PLUGINS: \u0026#34;true\u0026#34; deploy: # This section is for compatibility with Swarm resources: reservations: devices: - driver: nvidia count: all capabilities: [gpu] volumes: - ${APPDATA}/n8n/docling_artifacts:/models runtime: nvidia restart: always depends_on: docling-serve-initial: condition: service_completed_successfully First, I tried to get n8n to load all the papers from the arxiv-existing folder Lokman sent me. It contained 56 items, and it took a really long time to load. Next, I started poking around the docling API. Before unleashing it on all 56 papers, I tested it out with just two to see what would actually happen.\nI first tried all of the features that docling offers:\npipeline (str). The choice of which pipeline to use. Allowed values are standard and vlm. Defaults to standard. do_table_structure (bool): If enabled, the table structure will be extracted. Defaults to true. do_code_enrichment (bool): If enabled, perform OCR code enrichment. Defaults to false. do_formula_enrichment (bool): If enabled, perform formula OCR, return LaTeX code. Defaults to false. do_picture_classification (bool): If enabled, classify pictures in documents. Defaults to false. do_picture_description (bool): If enabled, describe pictures in documents. Defaults to false. However, it took a ridiculous amount of time and memory to run, and it was prone to crashing. After digging around, I found out that other people were hitting this same issue. The formula and code awareness models, while small, will apparently eat your entire GPU VRAM. So I had to turn off all the enrichments.\nPart 3: The n8n Orchestration Nightmare # This was my first time really using n8n for a complex batch job, and honestly, it was frustrating.\nProblem 1: The ZIP File docling-serve returns a ZIP file with paper.md and an artifacts/ folder. n8n\u0026rsquo;s \u0026ldquo;Decompress\u0026rdquo; node doesn\u0026rsquo;t output a nice array you can loop over. It\u0026rsquo;s\u0026hellip; not an array. It\u0026rsquo;s a single item with multiple binary properties. After digging through forums, I found you have to use a custom Code node to manually split it:\n// This code is required to split a single item with multiple binary files // into multiple items, each with one binary file. let results = []; for (item of items) { for (key of Object.keys(item.binary)) { results.push({ json: { fileName: item.binary[key].fileName }, binary: { data: item.binary[key] } }); } } return results; Problem 2: Nested Loops are Buggy My next instinct was one giant workflow:\nLoop over all 50 PDFs. (Nested) Call Docling API. (Nested) Get the ZIP, run the code above. (Nested) Loop over the resulting files to save them. This failed miserably. Data from the first paper\u0026rsquo;s loop would \u0026ldquo;bleed\u0026rdquo; into the second paper\u0026rsquo;s execution. It would just process the first paper over and over.\nThe Solution: Master/Child Workflows The only stable solution was to refactor:\nMaster Workflow: Its only job is to loop through the 50 PDFs and call a \u0026ldquo;Child\u0026rdquo; workflow once per paper. Child Workflow: Receives one paper, calls Docling, saves the files, and finishes. This isolated the execution and finally worked. I also created a merge node to filter out previously processed items, in case I wanted to run the workflow again in the future.\nThis process might look easy, but it took me several hours. At first, I didn\u0026rsquo;t know the Edit Fields node existed, so I was comparing the filenames directly, without realizing the file extensions were different (silly me). I had to modify the items in the output node, using an expression to cut the .md extension and replace it with .pdf just to get the filter to work.\nIn the end, it processed 39 papers in 11 minutes (with table enrichment on). A success\u0026hellip; but it felt way too complicated.\nPart 4: The Final Pivot - From n8n Hell to Python # The next step was extracting metadata from these new Markdown files.\nI briefly tried pdfVector because of its JSON schema feature. Then I saw the price: 2 credits per page. A 25-page paper would cost 75 credits. It was a complete non-starter. The results was actually good tho. At least I can utilize the JSON schema of it.\n{ \u0026#34;type\u0026#34;: \u0026#34;object\u0026#34;, \u0026#34;properties\u0026#34;: { \u0026#34;title\u0026#34;: { \u0026#34;type\u0026#34;: \u0026#34;string\u0026#34;, \u0026#34;description\u0026#34;: \u0026#34;Title of the research paper\u0026#34; }, \u0026#34;type\u0026#34;: { \u0026#34;type\u0026#34;: \u0026#34;string\u0026#34;, \u0026#34;description\u0026#34;: \u0026#34;Type of the paper\u0026#34;, \u0026#34;enum\u0026#34;: [ \u0026#34;method\u0026#34;, \u0026#34;benchmark\u0026#34;, \u0026#34;dataset\u0026#34;, \u0026#34;application\u0026#34;, \u0026#34;survey\u0026#34;, \u0026#34;other\u0026#34; ] }, \u0026#34;categories\u0026#34;: { \u0026#34;type\u0026#34;: \u0026#34;array\u0026#34;, \u0026#34;description\u0026#34;: \u0026#34;A list of the paper\u0026#39;s methodology.\u0026#34;, \u0026#34;items\u0026#34;: { \u0026#34;type\u0026#34;: \u0026#34;string\u0026#34;, \u0026#34;enum\u0026#34;: [ \u0026#34;Weakly Supervised\u0026#34;, \u0026#34;Semi Supervised\u0026#34;, \u0026#34;Training Free\u0026#34;, \u0026#34;Instruction Tuning\u0026#34;, \u0026#34;Unsupervised\u0026#34;, \u0026#34;Hybrid\u0026#34; ] } }, \u0026#34;github_link\u0026#34;: { \u0026#34;type\u0026#34;: \u0026#34;string\u0026#34;, \u0026#34;format\u0026#34;: \u0026#34;url\u0026#34;, \u0026#34;description\u0026#34;: \u0026#34;Link to the GitHub repository (if available)\u0026#34;, \u0026#34;nullable\u0026#34;: true }, \u0026#34;summary\u0026#34;: { \u0026#34;type\u0026#34;: \u0026#34;string\u0026#34;, \u0026#34;description\u0026#34;: \u0026#34;Description of the novelty of the paper\u0026#34; }, \u0026#34;benchmarks\u0026#34;: { \u0026#34;type\u0026#34;: \u0026#34;array\u0026#34;, \u0026#34;description\u0026#34;: \u0026#34;A list of benchmarks used in the paper\u0026#34;, \u0026#34;items\u0026#34;: { \u0026#34;type\u0026#34;: \u0026#34;string\u0026#34;, \u0026#34;enum\u0026#34;: [ \u0026#34;cuhk-avenue\u0026#34;, \u0026#34;shanghaitech\u0026#34;, \u0026#34;xd-violence\u0026#34;, \u0026#34;ubnormal\u0026#34;, \u0026#34;ucf-crime\u0026#34;, \u0026#34;ucsd-ped\u0026#34;, \u0026#34;other\u0026#34; ] }, \u0026#34;nullable\u0026#34;: true }, \u0026#34;authors\u0026#34;: { \u0026#34;type\u0026#34;: \u0026#34;array\u0026#34;, \u0026#34;description\u0026#34;: \u0026#34;A list of the paper\u0026#39;s authors\u0026#34;, \u0026#34;items\u0026#34;: { \u0026#34;type\u0026#34;: \u0026#34;string\u0026#34; } }, \u0026#34;date\u0026#34;: { \u0026#34;type\u0026#34;: \u0026#34;string\u0026#34;, \u0026#34;format\u0026#34;: \u0026#34;date\u0026#34;, \u0026#34;description\u0026#34;: \u0026#34;Publication date of the paper (YYYY-MM-DD)\u0026#34; } }, \u0026#34;required\u0026#34;: [ \u0026#34;title\u0026#34;, \u0026#34;type\u0026#34;, \u0026#34;categories\u0026#34;, \u0026#34;date\u0026#34;, \u0026#34;summary\u0026#34; ], \u0026#34;additionalProperties\u0026#34;: false } So I decided to roll my own json extractor on n8n. I added an \u0026ldquo;Information Extractor\u0026rdquo; node to my n8n workflow, using the JSON schema I\u0026rsquo;d built. Again, the local model disappointed me with inferior results.\nLocal qwen3-4b: Failed to call the tool. OpenRouter gpt-4.1-nano: Worked perfectly. The final nail in the n8n-coffin came when I tried to set up my Hugo site. To make images work, Hugo needs a \u0026ldquo;Page Bundle\u0026rdquo; structure:\n- content/ ----papers/ -------\u0026lt;paper-name\u0026gt;/ -----------index.md -----------artifacts/ ---------------image.png Trying to make n8n create this dynamic directory (\u0026lt;paper-name\u0026gt;) and rename the file to index.md was a joke. I was writing crazy expressions, using Execute Command nodes, Edit Fields nodes\u0026hellip; it was just a mess.\nI was so frustrated with how n8n handles basic file and path manipulation that I just gave up on it for orchestration.\nThe Final Architecture: Python as the Orchestrator # I threw away the complex \u0026ldquo;Master\u0026rdquo; n8n workflow and replaced it with a single Python script.\nThis new hybrid architecture is the best of all worlds:\nPython (main.py) is the \u0026ldquo;Master.\u0026rdquo; It\u0026rsquo;s simple, I can debug it, and it handles file I/O perfectly. Docker runs my services (docling-serve, n8n, mcp-server). n8n is now just a simple \u0026ldquo;serverless function.\u0026rdquo; It\u0026rsquo;s a single webhook that receives a file path, extracts the JSON, and sends it back. Here\u0026rsquo;s the new flow:\nPython script loops through all PDFs. Calls docling-serve (running in Docker) to get the ZIP. Unzips the files locally into the correct Hugo Page Bundle structure (\u0026lt;paper-name\u0026gt;/index.md). Calls the n8n webhook with the absolute path to the new index.md. The n8n workflow reads the file, extracts the JSON using the LLM, and sends the JSON back as the webhook response. Python receives the JSON, formats it with ruamel.yaml, and writes it as front matter to the top of the index.md. The script moves the original PDF to a done folder. This journey was\u0026hellip; a lot. But the final system is clean, robust, and a perfect example of using the right tool for the right job—even if it takes a few frustrating detours to find it.\n","date":"2 November 2025","externalUrl":null,"permalink":"/posts/arxiv-n8n-docling/","section":"Posts","summary":"This is my story of how I attempt to build an AI-powered pipeline for ArXiv papers. It was a journey that started with a cool idea about AI agents and ended with me wrestling Docker, n8n, and Python into submission.\n","title":"Building an AI ArXiv Pipeline: Is n8n the Future of Automation? Not Quite","type":"posts"},{"content":"In the previous parts of this series, we went from a single hand-me-down PC to a full 3-node cluster, and then we installed Proxmox as our hypervisor. We even managed to pass through a GPU to a VM for near-native performance.\nThis time, we\u0026rsquo;re diving into one of the most critical parts of any homelab: storage.\nIn fact, many people (myself included) start this homelab journey just because we need a platform to store data other than Google Drive as we creep toward that 15GB limit. Now, imagine having your own private Google Drive at home, but with 10s of terabytes.\nWe will be virtualizing TrueNAS, a world-class, open-source NAS (Network Attached Storage) platform, right inside Proxmox. We\u0026rsquo;ll pass through our physical hard drives, set up a ZFS storage pool, and create network shares (like NFS and SMB) that our entire homelab will use.\nSo, what is a NAS # But first, to understand what a NAS is, you have to consider the core problem it solves.\nImagine you\u0026rsquo;re a video editor, working on your local machine with 10GB of footage. It\u0026rsquo;s going smoothly\u0026hellip; as long as you\u0026rsquo;re working alone.\nBut what happens when the workload is heavy and you need to collaborate with other editors? Are you going to wait for someone to finish, copy all 10GB of files onto a USB drive, and walk it over to them? This could take an hour. Then you do your small part, and you have to transfer it back for another hour. This painful, time-wasting process is a complete nightmare. Not to mention, if someone\u0026rsquo;s computer has problems and their files are gone, how can you restore them?\nThis is where a NAS comes to the rescue. A NAS device is essentially a smart server full of hard drives that plugs directly into your network. Instead of everyone having their own 10GB copy, the single, master copy of the project lives on the NAS. Now, everyone on the team can connect to the NAS over the fast local network at the same time. It appears on their computer just like another hard drive.\nYou, the other editor, and the animator can all read and write to the same central files simultaneously. There\u0026rsquo;s no more copying, no more hour-long USB transfers, and no more confusion about \u0026ldquo;Which file is the latest version?\u0026rdquo; No more worrying about lost files if your computer has problems, as long as the central NAS is still active. Now, in our homelab context, replace \u0026ldquo;editors\u0026rdquo; with \u0026ldquo;VMs\u0026rdquo; or \u0026ldquo;PCs\u0026rdquo;. The number isn\u0026rsquo;t three, but maybe 10 or 100. You will save a bunch of time by having a NAS so your VMs can access data efficiently, without relying on their own local storage in case one of them fails.\nWhich NAS platform to choose # I used to be a Synology user, running Xpenology to get the DSM (Disk Station Manager) platform on my non-Synology hardware. After using it for a while, although the software support is top-notch and very user-friendly, I migrated to TrueNAS because I wanted deeper control and customization. The OS TrueNAS provides also feels lighter to me and seems to be more focused and specialized as a storage platform. Synology, on the other hand, seems more like an All-in-One platform, encompassing a hypervisor and integrated apps, which feels a bit bloated for my needs.\nOne more thing: TrueNAS officially supports ZFS and Synology officially supports BTRFS as a file system. From what I understand, BTRFS is for flexibility and less demanding roles, while ZFS is for maximum reliability and data integrity, especially in servers. I have to admit that I don\u0026rsquo;t use all the advanced features of either, but with the correct configuration (log, cache, etc.), ZFS can outperform BTRFS in both speed and scale.\nVirtualize NAS \u0026hellip; why not? # Preface: For best practices, and if you have the money, get yourself a dedicated NAS device like a Synology.\nIn theory, dedicating a machine to be a NAS means I shouldn\u0026rsquo;t run other compute resources on it, as it breaks the \u0026ldquo;specialized tool\u0026rdquo; idea I mentioned before. You should have a separate Synology or custom NAS build, separate from your compute resources like Proxmox.\nAt first, I thought about using one of the other machines (HP Prodesk or Dell Optiplex) as the main NAS device, without Proxmox installed. But a NAS requires lots of HDD slots; otherwise, what\u0026rsquo;s the point? I quickly realized those SFF computers only have one or two HDD slots. The only computer I have that allows multiple slots is the Lenovo P520.\nBut the Lenovo P520 is the strongest of all; sacrificing it only to store data would be so wasteful. Since I\u0026rsquo;m still a student, I don\u0026rsquo;t want to spend more money and wanted to consider an alternative.\nLuckily, with Proxmox, I can virtualize the NAS platform in a VM and pass through the HDDs directly to it. This isn\u0026rsquo;t \u0026ldquo;best practice\u0026rdquo; because it creates a single point of failure: if the Lenovo P520 is in maintenance mode, everything else loses access to storage. I only chose this because it allows me to use the PC\u0026rsquo;s multiple HDD slots while still running other compute resources, all without breaking my bank. With proper HDD passthrough, we give the TrueNAS VM direct, bare-metal control of the drives. The performance is basically native, and it\u0026rsquo;s been rock-solid for me.\nThis is almost the same process we used for the GPU. We need to tell Proxmox, \u0026ldquo;Don\u0026rsquo;t touch these drives; give them directly to this VM.\nFirst, we need the unique IDs for our drives. Don\u0026rsquo;t use /dev/sda or /dev/sdb, as those can change on reboot. We need the permanent \u0026ldquo;by-id\u0026rdquo; path. Hop on to the pve/Disks tab to see the information and take note of the disk serial numbers you want to pass through. Then, in the shell, run:\nls -l /dev/disk/by-id/ Look for the ata- names that match the serial numbers of your whole disks (e.g., ata-WDCWD10EZEX-00WN4A0...). You want the main disk ID, not the ...-part1 or ...-part2 names.\nOnce you have your disk IDs, make sure your TrueNAS VM is shut down. Then, run this command for each disk:\n# qm set \u0026lt;VMID\u0026gt; --\u0026lt;controller\u0026gt; \u0026lt;controllernumber\u0026gt; /dev/disk/by-id/\u0026lt;your-disk-id\u0026gt; # Example for the first disk: qm set 100 --scsi1 /dev/disk/by-id/ata-WDCWD10EZEX-00WN4A0WD-WCC6Y... # Example for the second disk: qm set 100 --scsi2 /dev/disk/by-id/ata-WDCWD10EZEX-00WN4A0WD-WCC6Z... Repeat this for all your drives, incrementing the scsi number. Now, when you boot your TrueNAS VM, it will see these disks as if they were physically attached. I decided to reserve only my 4TB + 1TB HDD drives for TrueNAS; the remaining HDD is used for the Proxmox OS itself.\nRAID Theory (but I use Stripe, btw) # Preface: Again, for best practice, don\u0026rsquo;t be like me.\nAfter you\u0026rsquo;ve passed through the drives and booted TrueNAS, you\u0026rsquo;ll go to Storage \u0026gt; Pools \u0026gt; Create Pool. This is where you make the most important decision: how to combine your drives.\nRAID (Redundant Array of Independent Disks) combines multiple drives into one logical unit. ZFS uses its own software-based RAID, with a few main types:\nStripe (RAID 0): This is for max speed and max risk. It \u0026ldquo;stripes\u0026rdquo; data across all drives.\nPros: Super-fast reads/writes. You get 100% of the storage (e.g., 4TB + 1TB = 5TB).\nCons: If one drive fails, all data on all drives is gone. Zero redundancy.\nMirror (RAID 1): This is for max safety. It\u0026rsquo;s a 1-to-1 clone (e.g., two 4TB drives mirrored = 4TB usable).\nPros: If one drive fails, your data is 100% safe on the other.\nCons: You only get 50% of your total storage.\nRAIDZ1 (RAID 5): The classic balance. It stripes data and writes a \u0026ldquo;parity\u0026rdquo; block, which can be used to rebuild a lost drive.\nPros: Good performance and safety. You can lose one drive.\nCons: You lose the capacity of one drive to parity (e.g., three 4TB drives = 8TB usable).\nRAIDZ2 (RAID 6): Same as above, but with two parity blocks. You can lose two drives.\nSo you might guess I should be choosing Mirror or RAIDZ1, right? Well, I chose Stripe.\nI know what you\u0026rsquo;re thinking: \u0026ldquo;That\u0026rsquo;s insane! You have no redundancy!\u0026rdquo; And you\u0026rsquo;re right. I\u0026rsquo;m fearless, and I honestly don\u0026rsquo;t care about a hardware failure.\nMy philosophy is that my data is the most important thing, not the hardware. I already have a 5TB Microsoft 365 (OneDrive) plan that backs up 100% of my critical data. If a drive in my Stripe pool fails, who cares? The data itself is safe in the cloud. I\u0026rsquo;ll just buy a new drive, create a new pool, and restore from my backup.\nThis is my personal trade-off. I lose time (it will be a pain to wait for restoration). But I gain money and speed, as I don\u0026rsquo;t have to buy extra HDDs just for parity, and I get 100% of my storage capacity and maximum performance. But seriously, do not do this in production.\nSo, I went ahead and created a Stripe pool that combines my 4TB and 1TB drives. Additionally, I also created a 128GB SLOG by passing through an NVMe drive (a similar process to the GPU, as it\u0026rsquo;s a PCI device). Since I\u0026rsquo;m going to use this NAS as file storage for other VMs and containers, I need a SLOG to improve performance for synchronous writes by storing them in a fast log before they\u0026rsquo;re written to the slower pool drives. You don\u0026rsquo;t need this if you only use your NAS for storing media.\nNext, after creating a pool (in my case, storage), we need to create datasets for better isolation. For example, I\u0026rsquo;ll use TrueNAS for:\nappdata: storing application data for containers\niso: storing PS4, Switch, and Windows games\nmedia: storing media files for Jellyfin, manga for Suwayomi, etc.\nproxmox: storing backups, ISO images, and CT templates for Proxmox\nminio: storing S3 object storage\nTherefore, I\u0026rsquo;ll create these 5 different datasets in my pool.\nFile sharing # Now that we have our pool and datasets, the next step is to share them over the network. There are three file-sharing protocols I currently use: SMB, NFS, and FTP.\nNFS for appdata # As I mentioned in the previous part, I primarily use TrueNAS to store persistent data for my stateful applications (containers). This way, even if I delete and recreate a container, it still functions the same as before.\nNFS is the native file-sharing protocol for Linux/Unix environments. Since all my containers and Proxmox are Linux-based, it\u0026rsquo;s the best tool for the job.\nFirst, set up NFS on TrueNAS via the Shares tab. The main thing is to enable NFSv4 support. Next, for the appdata dataset share, go to Advanced Options and change the Mapall User and Mapall Group to root.\nThis maps any request from a non-root user (e.g., user 1000 inside a container) to the root user on TrueNAS. This ensures that when a container tries to write data, TrueNAS grants it the required permissions. Don\u0026rsquo;t do this in production, but in a homelab, you can mitigate the risk by limiting the client IPs that can connect (in the Hosts section).\nAfter configuring TrueNAS, we SSH into our Docker VM to mount this NFS storage on boot.\n# Install package for nfs client sudo apt update \u0026amp;\u0026amp; sudo apt install nfs-common # Create a mount point in local directory sudo mkdir -p /mnt/storage/appdata # Open the configuration file for drive mount sudo vim /etc/fstab Go to the bottom of the file and add a new line, like this example:\n10.69.1.102:/mnt/storage/appdata /mnt/storage/appdata nfs defaults,_netdev,nofail 0 0 Notice these three flags after the nfs block:\ndefaults: Uses standard options (like rw for read-write and auto to mount at boot). _netdev: This tells your system \u0026ldquo;this is a network device, wait for the network to be online before trying to mount it.\u0026rdquo; This prevents your system from hanging at boot. nofail: This tells your system \u0026ldquo;if you can\u0026rsquo;t mount this for any reason (e.g., the NFS server is offline), don\u0026rsquo;t hang the boot process. Just skip it and continue.\u0026rdquo; After that, run these commands to reload the config and mount the share:\nsudo systemctl daemon-reload sudo systemctl restart remote-fs.target After verifying the mount, we tell Docker to use it as a local volume by creating an APPDATA environment variable and referencing it in our docker-compose.yml files.\nNFS for proxmox # I use this exact same NFS protocol for the proxmox dataset, which I add as a storage target in the Proxmox datacenter. This is especially useful for a Proxmox cluster: once you set it up on one node, the storage is shared with all nodes. This allows for instant live migration of VMs, as you\u0026rsquo;re only moving the RAM, not the disk.\nHere\u0026rsquo;s the example demo of live migration VMs on Proxmox using shared storage from TrueNAS.\nSmb for laptop, phone clients # If you don\u0026rsquo;t want to install an app like Jellyfin, you can still watch your movies by using SMB to mount the media dataset directly in your laptop\u0026rsquo;s file browser. You can open, edit, and delete files as if it were a local folder. This is the same protocol the video editors in my earlier example would use.\nBelow is a showcase where I can watch a movie on both my laptop and phone by just opening the file, no download needed.\nFTP for edge devices like PS4 and Nintendo Switch # For devices that don\u0026rsquo;t support NFS or SMB (like my hacked PS4 and Nintendo Switch), no problem. TrueNAS also offers the FTP protocol. I use it to install games on my console. I can easily download a game via qbittorrent (running on the NAS) and then connect the device to my FTP server to transfer the file.\nAdditional quality-of-life applications # Remember when I said it\u0026rsquo;s \u0026ldquo;best practice\u0026rdquo; to separate compute from storage? Well, your life will be much easier if you install a few lightweight, storage-related applications on your NAS.\nDownload clients # Firstly, aria2 is a lightweight, multi-protocol, cross-platform download utility that operates on the command line. When bundled with a web UI like aria2-ng, it makes downloading to your NAS so much easier. There are browser extensions that can even capture download links and send them directly to your aria2 instance, automating file downloads right onto your NAS.\nSecondly, if you\u0026rsquo;ve been \u0026ldquo;sailing the seas\u0026rdquo; for a long time, you\u0026rsquo;ve definitely heard of uTorrent, which is a client to download\u0026hellip; \u0026ldquo;Linux ISOs\u0026rdquo;\u0026hellip; from a decentralized peer-to-peer network. However, uTorrent only provides a desktop application and is full of ads, so the community has switched to qbittorrent. It has a great web interface and supports being deployed as a Docker image, so I\u0026rsquo;ve been using it to download my \u0026ldquo;Linux ISOs\u0026rdquo; for a long time now.\nS3-compatible storage # I\u0026rsquo;ve just started playing around with this recently, but I can already see its tremendous potential. If you don\u0026rsquo;t know, S3 is a cloud storage service from AWS that allows you to store data as objects (a file plus metadata) organized into \u0026ldquo;buckets.\u0026rdquo; It\u0026rsquo;s used for a wide range of applications, including backups, data lakes, and hosting static websites.\nWith the recent AWS outage, for anyone wanting to migrate to an alternative, self-hosted object storage platform, I recommend Minio. It\u0026rsquo;s open-source and fully S3-compatible. I\u0026rsquo;m definitely going to use this for my Terraform remote state, which I\u0026rsquo;ll introduce in the next part.\nAnd that\u0026rsquo;s all the applications I\u0026rsquo;ve installed on my TrueNAS server. As you can see, there aren\u0026rsquo;t many, and they consume very little RAM, but they bring so much quality-of-life value.\nJust so you know, if you want to deploy any app, TrueNAS also has a dedicated interface in the \u0026ldquo;Apps\u0026rdquo; tab. You\u0026rsquo;ll likely just need to configure a dataset as an \u0026ldquo;app\u0026rdquo; type and set the user/group for the application to 568, which is the default ID for the built-in \u0026ldquo;apps\u0026rdquo; user (yours might look different).\nBackup # This is critical. Who knows when your homelab will suddenly die? (wink, wink).\nThis section even deserves its own article. I\u0026rsquo;ve switched between many backup solutions in the past: from only using Proxmox and rclone-to-OneDrive, to using Restic inside a Docker VM, and finally to using the integrated TrueNAS data protection features. I\u0026rsquo;m quite satisfied with the current setup, as it offers minimal restore time, minimal storage overhead, and easy progress tracking to see if anything went wrong.\nHere\u0026rsquo;s my strategy:\n1. Proxmox Backups First, for Proxmox, I enabled the integrated backup feature. I have it scheduled to store backups of all my VMs and LXCs directly to the proxmox dataset (via NFS) on the TrueNAS server we just set up.\n2. Application Data Backups Secondly, for my application data, I don\u0026rsquo;t need a separate tool like Restic. Since I designed all my containers to store their persistent data directly on the appdata NFS share, the data is already centralized on the NAS. Backing up the NAS is backing up the apps.\n3. Local Snapshots (The First Copy) Thirdly, now that I have all my centralized data (Proxmox backups and appdata) on TrueNAS, I enable snapshots. According to the 3-2-1 backup rule, you need at least 3 copies of your data. TrueNAS helps with this: it has the ability to configure automatic snapshots every night and allows you to restore even a single file or subfolder from those snapshots easily.\n4. Off-site Backup (The \u0026ldquo;3-2-1\u0026rdquo; Completion) Finally, the 3-2-1 rule states you need 3 copies, on 2 different media types, with 1 copy off-site.\nAgain, I don\u0026rsquo;t have the budget for a second on-site backup machine, so I\u0026rsquo;m relying on my 1 off-site copy. I\u0026rsquo;ve had a 5TB OneDrive plan (for free!) for a long time, so I\u0026rsquo;ve been using it as my primary off-site backup, and I still haven\u0026rsquo;t had any issues with it, so yeah.\nTrueNAS has also made configuring rclone to connect with OneDrive much easier through its web interface. No need to remember all the quirky command-line flags anymore; you just need to toggle settings and click a few buttons to have it all configured. You can also view logs if a task fails and get notified via email immediately.\nSummary # And that\u0026rsquo;s it! We now have a rock-solid, (mostly) independent, and high-performance NAS running as a VM inside Proxmox.\nWe passed through our HDDs for bare-metal performance, explored the trade-offs of RAID (and my own \u0026ldquo;fearless\u0026rdquo; Stripe setup), and created SMB and NFS shares for every device in our lab.\nThis NFS share is the final, critical piece of our infrastructure. We have prepared for next part where we create our Kubernetes cluster. With this design, we won\u0026rsquo;t store any persistent data inside the cluster\u0026rsquo;s VMs. We\u0026rsquo;ll tell them to store everything on our new TrueNAS server. If a Kubernetes VM dies, it\u0026rsquo;s no big deal - we just spin up another, connect it to the NFS share, and all our application data is still right where we left it.\n","date":"2 November 2025","externalUrl":null,"permalink":"/posts/on-premise-storage/","section":"Posts","summary":"In the previous parts of this series, we went from a single hand-me-down PC to a full 3-node cluster, and then we installed Proxmox as our hypervisor. We even managed to pass through a GPU to a VM for near-native performance.\n","title":"On-Premise 101 (Part 3): My \"Fearless\" NAS Build with Virtualized TrueNAS, ZFS, and Cloud Backups","type":"posts"},{"content":"","date":"2 November 2025","externalUrl":"https://github.com/phuchoang2603/sis-arxiv-vad-papers","permalink":"/projects/sis-arxiv-vad-papers/","section":"Projects","summary":"This project is a comprehensive platform for managing, processing, and displaying ArXiv research papers. It combines a Hugo static site with a powerful backend of containerized services for AI-driven PDF processing, metadata extraction, and ArXiv interaction.","title":"SIS ArXiv VAD Papers","type":"projects"},{"content":"In the previous part, we covered how I started playing around with my hand-me-down computer and then escalated to building a whole 3-node cluster. It\u0026rsquo;s been a 3-year journey, and there have been so many changes to the software stack I host. But no matter what, there\u0026rsquo;s one thing I\u0026rsquo;ve used consistently from beginning to end: Proxmox (Hypervisor). So before we dive into how I deploy my applications on Docker or Kubernetes, I\u0026rsquo;ll show you the glue that connects the hardware we just built to all the software we\u0026rsquo;re going to deploy.\nWhy Proxmox? The Magic of Type 1 Hypervisors # Many of you have probably heard the term Virtual Machines (VMs), the ones that programs like VirtualBox and VMWare create. Those programs, which are often called a Type 2 Hypervisor, let you basically run a whole Guest OS (i.e., Windows, Linux, \u0026hellip;) on top of your current one (which is called the Host OS). Because it doesn\u0026rsquo;t actually use the machine\u0026rsquo;s hardware resources directly - it goes through a translation layer in the Host OS - it lets you do anything inside without worrying about the machine being compromised. Therefore, it\u0026rsquo;s frequently used for testing out dangerous programs like Honorlock online exam proctoring (oops, don\u0026rsquo;t arrest me prof Eggers). It runs fine with basic, lightweight programs, sure. But if you want to push it harder to run heavier ones, it might not utilize all the potential performance the machine has.\nThat\u0026rsquo;s where the Type 1 Hypervisor comes in. It can still create Virtual machines like above. But right now, notice it no longer has the Host OS between it and the Hardware resources. That means it can now directly access the full performance of the machine, without relying on the Host OS\u0026rsquo;s translation layer, making the Virtual machine running on top of it much more efficient. Maybe you don\u0026rsquo;t know this, but when you enable Windows Subsystem for Linux 2 (WSL2), Windows re-configures itself to run on top of its own Type 1 Hypervisor called Hyper-V. WSL2 then runs as a separate, lightweight VM directly on that same hypervisor, side-by-side with Windows, providing full system call compatibility. Other examples include VMWare ESX or ESXi, and notably, Proxmox.\nProxmox is an open-source virtualization management solution that uses KVM to manage virtual machines and LXC to manage containers. It is a Type 1 Hypervisor that built on a Debian-based Linux distribution, and is managed through a central web interface for tasks like clustering, high availability, storage, and networking.\nSo that explains Proxmox. For me, the biggest benefit of using Proxmox was to easily pass through computer resources to bypass even the Hypervisor layer itself, eliminating significant overhead and allowing the VM to receive near-native hardware performance. This allows me to pass through a GPU into a VM to run inference on LLM models at the highest VRAM usage and computation; or pass through HDDs to a VM to use as a NAS (more on this in the next part). Additionally, as mentioned above, you get a beautiful web interface to manage the VMs and other useful features, so it has the potential to replace even Kubernetes or TrueNAS. However, I prefer specialization over convenience. I\u0026rsquo;d rather use a dedicated tool that does one job perfectly than a single, all-in-one tool that does multiple jobs just \u0026lsquo;okay\u0026rsquo;.\nTo demonstrate the ability of Proxmox, I will show you a guide to create a VM that is capable of directly inheriting a Nvidia GPU from the host machine, allowing the applications running inside the VM to utilize its fullest performance. However, note that this is just one of many examples you can do with Proxmox. In the next part, I will show you how to even virtualize a NAS device on it, or provision multiple VMs in one command.\nGetting Started: My Proxmox Post-Install Toolkit # I\u0026rsquo;m not going to cover the in-depth guide to install Proxmox as it\u0026rsquo;s similar to installing any other OS: just get the ISO file, burn it to a USB, turn off the computer, boot from the USB, and follow the installation guide (the default options are pretty solid). The settings they ask you to configure from the beginning can also be changed later after installation. So no need to be afraid of anything.\nNext, I will show you commands that I actually use after installing Proxmox, as I believe they are really useful.\nThis script provides options for managing Proxmox repositories, including disabling the Enterprise Repo, adding or correcting PVE sources, enabling the No-Subscription Repo, adding the test Repo, disabling the subscription nag, updating Proxmox VE, and rebooting the system.\nbash -c \u0026#34;$(curl -fsSL https://raw.githubusercontent.com/community-scripts/`Proxmox`VE/main/tools/pve/post-pve-install.sh)\u0026#34; To create a new Proxmox VE Ubuntu 24.04 VM, run the command below in the Proxmox Shell. Note that if you want to use pass-through features like a GPU, you must select q35 as the machine type, host as the CPU type, and make sure not to start the machine after installation. In the cloud-init tab, change the username, password, network address, and SSH public key so you can SSH into it later.\nbash -c \u0026#34;$(curl -fsSL https://raw.githubusercontent.com/community-scripts/`Proxmox`VE/main/vm/ubuntu2404-vm.sh)\u0026#34; This script is fantastic because it creates a cloud-init template, which is much faster than manually installing from an ISO. It lets you pre-configure your username, password, and networking, so the VM is ready to go in seconds. It\u0026rsquo;s the perfect way to spin up new VM s quickly, especially when you plan to make more for a Kubernetes cluster later. If you want more useful scripts like this, feel free to hop over to here.\nPassing a GPU to a VM # Firstly, we don\u0026rsquo;t want the Proxmox host system utilizing our GPU(s), so we need to blacklist the drivers. I\u0026rsquo;m not gonna provide the whole full guide, as it\u0026rsquo;s a really long process and requires you to know your system to adjust accordingly. But here\u0026rsquo;s the ultimate guide that provides detailed explanations and steps for doing so.\nSecondly, after we verify the GPU was passed through successfully, we need to set up the VM to receive it:\nAfter Ubuntu has finished installing and is reachable by SSH on your network, shut down the `VM and go to the \u0026ldquo;Hardware\u0026rdquo; tab. Click \u0026ldquo;Add\u0026rdquo; \u0026gt; \u0026ldquo;PCI Device\u0026rdquo;. Select \u0026ldquo;Raw Device\u0026rdquo; and find your GPU. Click the \u0026ldquo;Advanced\u0026rdquo; checkbox, \u0026ldquo;All Functions\u0026rdquo; checkbox, and \u0026ldquo;PCI-Express\u0026rdquo; checkbox, then hit Add. Start the VM again and type lspci in the console. Search for your GPU. If you see it, you\u0026rsquo;re good to go! Thirdly, we need to install the drivers inside the VM so that the software can utilize the GPU correctly. There are multiple ways to do this, but I always refer to this site to install.\nsudo apt update \u0026amp;\u0026amp; sudo apt install ubuntu-drivers-common # List drivers sudo ubuntu-drivers list --gpgpu # Let’s assume we want to install the `570-server` driver (listed as `nvidia-driver-570-server`): sudo ubuntu-drivers install --gpgpu nvidia:570-server #You will also want to install the following additional components: sudo apt install nvidia-utils-570-server # After that, reboot your system sudo reboot # Once the machine is back up, check to be sure your drivers are functioning properly nvidia-smi Installing Docker (And Why I Use a VM, Not LXC) # Note that Proxmox also has LXC, as an alternative to Docker. While LXC offers better performance, the software support and ease of use aren\u0026rsquo;t as good as Docker. Not to mention it is harder to pass through a full physical device (like a GPU) to an LXC container, because by design, it\u0026rsquo;s tightly integrated with the host kernel.\nTherefore, I only use LXC for critical software that needs to stay active alongside the machine, such as Tailscale or Cloudflared. For most other things, I use a VM with Docker to experiment and test. If you want to install Docker and Nvidia Container Toolkit on the VM to help containers leverage your GPU, you\u0026rsquo;ll want to run these commands:\n# Add Docker\u0026#39;s repostiory to apt sources sudo apt-get update sudo apt-get install ca-certificates curl gnupg sudo install -m 0755 -d /etc/apt/keyrings curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /etc/apt/keyrings/docker.gpg sudo chmod a+r /etc/apt/keyrings/docker.gpg echo \\ \u0026#34;deb [arch=\u0026#34;$(dpkg --print-architecture)\u0026#34; signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu \\ \u0026#34;$(. /etc/os-release \u0026amp;\u0026amp; echo \u0026#34;$VERSION_CODENAME\u0026#34;)\u0026#34; stable\u0026#34; | \\ sudo tee /etc/apt/sources.list.d/docker.list \u0026gt; /dev/null ## Add Nvidia Container Toolkit\u0026#39;s repostiory to apt sources curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey | sudo gpg --dearmor -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg \\ \u0026amp;\u0026amp; curl -s -L https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list | \\ sed \u0026#39;s#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g\u0026#39; | \\ sudo tee /etc/apt/sources.list.d/nvidia-container-toolkit.list ## Install sudo apt-get update sudo apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin sudo apt-get install -y nvidia-container-toolkit # Use docker without sudo, need to log out then back in to apply this sudo usermod -aG docker $USER # Configure the container runtime by using the nvidia-ctk command and restart docker sudo nvidia-ctk runtime configure --runtime=docker sudo systemctl restart docker After successful installation, this will test to make sure that the NVIDIA container toolkit can access the GPU correctly.\nsudo docker run --rm --runtime=nvidia --gpus all ubuntu nvidia-smi # You should see the same output as running nvidia-smi without Docker. Summary \u0026amp; What\u0026rsquo;s Next: Prepping for TrueNAS # And that finishes the whole step of setting up a VM on top of a Type 1 Hypervisor to use an Nvidia GPU with near-native performance. Now you can go ahead and install any Docker containers you want.\nBut I\u0026rsquo;m not doing that right away, because I want those containers to use NFS storage. That way, even if the VM stops working correctly, I can still spin up a new one without losing my persistent application data. This approach is also helpful if you want to migrate from Docker to Kubernetes, as it allows you to reuse the data by just pointing to the correct NFS path. To do this, we have to set up TrueNAS, a network-attached storage platform.\n","date":"30 October 2025","externalUrl":null,"permalink":"/posts/on-premise-hypervisor/","section":"Posts","summary":"In the previous part, we covered how I started playing around with my hand-me-down computer and then escalated to building a whole 3-node cluster. It’s been a 3-year journey, and there have been so many changes to the software stack I host. But no matter what, there’s one thing I’ve used consistently from beginning to end: Proxmox (Hypervisor). So before we dive into how I deploy my applications on Docker or Kubernetes, I’ll show you the glue that connects the hardware we just built to all the software we’re going to deploy.\n","title":"On-Premise 101 (Part 2): Proxmox, the \"Glue\" for My Homelab","type":"posts"},{"content":"You might have heard about the recent AWS outage that caused many services to go down. To think that half of the internet relies on a single service is crazy, even if it might be the best in its field. The engineers at AWS have done many things to prevent this, such as high availability, multi-region, zones, etc., but if a critical part of it still goes down, the whole thing goes down too. Therefore, if you don\u0026rsquo;t want to give all your money and trust to a single company, you might want to explore the on-premise option, which is self-hosting.\nBut deploying on-premise isn\u0026rsquo;t that easy, though. You have to spend time, effort, and money to configure the system for yourself—from PC specs, network performance, and security, to virtualization. All of that is not a one-day setup. But it also gives you the power to manage everything yourself, too.\nSo if you want to start exploring and build an infrastructure like the one I\u0026rsquo;m currently hosting, here\u0026rsquo;s the guide for you to do so. This is the first part of the series On-Premise 101. In this part, I\u0026rsquo;ll share my whole journey from the beginning: how I acquired the hardware, the decisions I made, and finally, how I wired it all together with networking to get it ready for the software.\nIn this part, you\u0026rsquo;ll see me refer to Proxmox a lot and might ask, \u0026ldquo;What the hell is it?\u0026rdquo; For now, just think of it as the platform that allows me to create virtual machines. I will cover it in-depth in the next part.\nThe Beginning: My First SFF Server and Its Limits # Looking back, I started this journey 3 years ago. At the time, I had an HP Prodesk 600 G4 SFF that my mother gave me; she picked it up from the warehouse at her office. It was an old computer, so I didn\u0026rsquo;t know what to do with it at the time, since I already had a PC that I could do anything with. But when I was browsing my YouTube feed, I saw a video talking about how a creator used his Synology NAS (Network Attached Storage) to replace Google Drive and help him collaborate with his video editors easily.\nI thought it was so cool that you could transfer a file that big from one computer to another, without any wires, almost instantly. I checked the specs of the NAS and found out it was nothing powerful—just a bunch of HDDs and storage. That\u0026rsquo;s when I realized I could probably do all of that with my old computer, too.\nIf you want to read what I was doing with my old computer at that time, feel free to read my initial self-hosting story.\nThe specs of my PC weren\u0026rsquo;t really powerful, but its iGPU was capable enough to run Frigate (an NVR camera detection service) and Jellyfin (for video transcoding) pretty well. Later on, I tried to upgrade the PC specs , such as plugging in a graphics card for LLM models and upgrading the RAM to 16GB. But since it was a Small-Form-Factor (SFF) PC, it had limited upgrade options.\nFirst, I tried plugging in a GTX 1030, and although it worked, I had to remove the side panel to make room for it. Second, I tried to modify the HDD holder to fit two SATA HDDs in the case, but after running it for a while, I noticed it got really hot because the two HDDs were stuck so close together. So, I was tempted to look for another PC.\nLesson Learned: Small Form Factor (SFF) PCs are a great, cheap way to start, but their limited space for upgrades (especially for GPUs and storage) means you might outgrow them fast.\u0026quot;\nThe First Big Upgrade: The Lenovo P520 Workstation # After hitting the bottlenecks of my computer and having to move to the US for my undergrad studies, I had to leave my entire homelab back in Vietnam. I could still access it via Tailscale (a VPN service, which I\u0026rsquo;ll share more about later), but the latency was really high, so I also wanted to buy a new one. Actually, at that time, ChatGPT and LLMs were also getting popular, so I wanted to get a GPU to help me \u0026ldquo;study\u0026rdquo; more in that field. Since I\u0026rsquo;m also a student, I set my budget to under $500 total.\nBrowsing through eBay, YouTube, and Reddit for suggestions, I settled on the Lenovo P520 as the PC. Initially, I wanted to build my own PC, but I realized that if I bought all new hardware, it would cost much more than $500, and the specs wouldn\u0026rsquo;t be powerful enough. So I wanted to choose pre-built enterprise hardware, similar to my HP Prodesk 600 G4 above. Instead of an SFF, I wanted a Full-Tower this time so that I could upgrade it more easily later on. There were options like the Dell Optiplex, HP Prodesk, HP Elitedesk, etc., but I decided to choose Lenovo because of its price-to-performance ratio. My favorite YouTuber, Hardware Haven, also released a video on this PC.\nWith only $274, I got:\nCPU: Intel Xeon W-2123 (4 cores, 8 threads, 3.60 GHz)\nRAM: 32 GB of 2666 MHz ECC DDR4\nStorage: 1TB SATA SSD and 4TB HDD\nAnd a bunch of ports and upgrade options on the motherboard\nAs for the GPU, there were also lots of options to choose from, but I felt the Nvidia RTX 3060 and Nvidia Tesla P40 were the strongest candidates. On one hand, the Nvidia Tesla P40 attracted me because it has 24GB of VRAM, and as you may know, VRAM capacity is king in the LLM world. But I was afraid the outdated software support would cost me in the future since it\u0026rsquo;s a card that was released nearly 10 years ago and uses the Pascal architecture. The RTX 3060, on the other hand, has 12GB of VRAM, was launched recently, and has active CUDA support, although I can\u0026rsquo;t run some of the largest LLM models on it. But with a price of only $200, and considering the fact that I might occasionally game on it, I decided to settle on the 3060.\nI also upgraded it to 64GB of ECC RAM with two purchases of 16GB sticks on eBay, because I also planned to use this machine as a NAS server (TrueNAS loves ECC RAM !). Later in the year, I also 3D-printed an HDD bay (the green part in the bottom right corner of the image below). This allowed me to install two additional 1TB HDDs, bringing the total NAS storage to 6TB.\nThe \u0026ldquo;Free PC\u0026rdquo; and My Dream of a 3-Node Cluster # A year later, which is just recently, I also received a new PC (notice I said \u0026ldquo;received\u0026rdquo;). The story of how I got this PC is really surprising. In the summer of 2025, I needed to go back to Vietnam to visit my family. Since it was a short trip, I decided not to bring the Lenovo P520 with me because it was really clunky, and at home, I still had the HP Prodesk 600 G4. But that summer was also the first time I learned Kubernetes and I really wanted to experiment with it. So I ran those experiments on the HP Prodesk 600 G4 with multiple virtual machines, and planned to upgrade to multiple physical nodes.\nWhen the summer ended, I planned to bring the HP Prodesk 600 G4 with me to the US. But unfortunately, after the flight, my machine would no longer boot up. Since I didn\u0026rsquo;t bring a monitor or any cables with me, I couldn\u0026rsquo;t troubleshoot the issue myself. I decided to bring it to the IT team at my school to hopefully fix it. After some back-and-forth discussion, they reached the conclusion that my machine had faulty RAM issues, and they only needed to plug out two 4 GB RAM sticks to boot it up normally.\nWhen I went to their office to pick up my device, I saw a bunch of old Dell Optiplex machines lying around, so I just asked what they planned to do with them. They answered that they were replacing them with newer devices for the library and planned to trash or give them away. I asked if I could have one for \u0026ldquo;study reasons,\u0026rdquo; and they happily gave me a Dell Optiplex 7020, and even threw in a Dell monitor. That\u0026rsquo;s the happiest day of my entire month since the HP Prodesk decided to leave me; not only I got it relived but I also welcomed a new one to my family ! The specs of the machine are:\nCPU: 3.6 GHz Intel Core i7-4790 Quad-Core\nRAM: 16GB of 1600 MHz DDR3\nStorage: 1TB HDD\nSo right now, I have 3 PCs, which is perfect for me to set up the Proxmox 3-node cluster I had dreamed of in the summer. But after a month of testing with it, I realized two of the nodes (the HP Prodesk and Dell Optiplex) constantly went offline. After unplugging and replugging the LAN cable, it would connect again, but only for a very short period. I figured this must be a networking bottleneck. And then, the upgrading journey continued.\nThe Bottleneck: Fixing a Failing Cluster with a Dedicated Network # If you want to know more about the background of my network design for my homelab, feel free to read this post.\nFor a long time while working with my homelab, I only had a single PC at one place at the time: the HP Prodesk when I was in Vietnam, and the Lenovo P520 when I was in the US. Therefore, I always connected the LAN cable from the PC directly to the router. But since I now have 3 PCs together at one place and plan to add a PS4 for remote gaming, relying on the ports on the router wouldn\u0026rsquo;t allow me to have more than 3 devices connected at the same time. Because of this, I got myself a basic 1 Gbps switch with 8 additional ports back in the summer.\nThe interesting thing about a switch is that not only does it expand your LAN ports, it\u0026rsquo;s also capable of helping the computers talk directly to each other, without having to go through the \u0026ldquo;middle-man\u0026rdquo;—the router. Think of it as a simple roundabout that connects all the local streets, with the big highway being the router. You can go from one street to another easily, and you can also get on the highway if you want to. This offloads the job from the router (which is like a toll booth on the highway), improving internal throughput and performance.\nHowever, with only one LAN port per computer, all my traffic was forced onto the same network. This means the high-speed Proxmox cluster communication (checking if nodes are online), VM traffic, and any storage traffic were all competing on the same, single 1Gbps \u0026ldquo;road.\u0026rdquo; This congestion was the root cause of my problem: critical cluster packets were being dropped, causing my nodes to think their partners were offline. Thanks for this comment from North Idaho Tom Jones on the Proxmox forum, I was able to find the solution:\nIMO - some things to look at your cluster communications :\n- How fast are your physical network interfaces ( 100-Meg , 1-Gig , 10-Gig , something faster ).\nThere is a possibility your interfaces might be busy moving I/O traffic to/from your VMs , and you don\u0026rsquo;t have the network additional I/O capacity for the cluster to communicate to the other cluster(s).\n- Are you performing backups when the cluster(s) drops ?\n- Do you have interface errors and/or packet drops on your physical \u0026amp; virtual ethernet interfaces ( and your external switch(es) ).\n- What is your current I/O bandwidth rate when you drop a node in your cluster ?\nI have a Proxmox network with 14-Clusters and 6-external NFS systems for my VM hard disk storage. I have 10 \u0026amp; 40 Gig network cards. My cluster IPs and my NFS IPs do not share any IP address space with my VMs \u0026mdash; My VMs \u0026amp; my Cluster IPs \u0026amp; my NFS IPs are on unique IP networks. I do this to keep unwanted/un-needed network chatter down to a minimum. I have never had a node drop out of the cluster , even with pushing 20+ Gig on multiple nodes in the cluster at the same time while all nodes are doing a backup at the same time.\nFrom this, I decided to upgrade my network infrastructure again, finding new network cards to expand the LAN ports on my PCs. I initially looked at 10 Gig, but after seeing the price of both the network cards and the switch, I immediately retreated. Although my switch is only 1 Gig, I didn\u0026rsquo;t plan to replace it right now because even 2.5 Gig switches are so expensive. So I opted for 2.5 Gig network cards for each computer, with the plan to upgrade the switch later.\nAt first, I was just looking for the cheapest network card. After searching on Amazon, I found this one, which is only $8 for a 2.5 Gig PCIe network card. I immediately bought 3 of them. When they arrived, I found out that the PCIe connection here is an M+B key, which is a totally different connector than an NVME drive or a standard PCI x1 slot. It\u0026rsquo;s a rare port for a PC to have, as it\u0026rsquo;s usually used for WiFi cards or Coral TPU accelerators.\nNeedless to say, only the HP Prodesk had this port, and it didn\u0026rsquo;t even look nice when plugged in (see the extra cord that\u0026rsquo;s totally detached from the case in the image above). So I had to return two of the network cards and buy two more. This time, I learned that I needed to find a specific PCI x1 network card, so I filtered carefully and bought this one. After they arrived, they fit nicely into the cases.\nThe next step, after wiring everything and connecting it to the switch, was to go into Proxmox to verify the PCs recognized the cards. Then I created a new subnet (192.168.69.0/24) just for these 3 PCs so they could talk to each other directly for cluster communication. After that, I changed the /etc/hosts file on each node so they would use these new IPs to communicate with each other. And after everything was done, voila! The cluster is now stable, and each node has two different addresses: one for the main network (via the router) and one for direct, high-speed communication with other nodes.\nWhat\u0026rsquo;s Next: Building the Software Stack (NAS, Kubernetes, and ArgoCD) # And with that, the hardware journey is complete. It\u0026rsquo;s been a fantastic learning experience, evolving from a single, hand-me-down HP Prodesk to a powerful 3-node Proxmox cluster.\nThis process taught me some valuable lessons:\nStart with what you have: That first SFF PC was the perfect, low-cost entry point into self-hosting.\nPlan for your bottlenecks: I quickly hit the physical limits of SFF (power, cooling, and PCIe slots). Moving to a full-tower workstation like the Lenovo P520 was the right call, giving me room for a real GPU and more storage.\nNetworking is critical for clusters: A \u0026ldquo;stable\u0026rdquo; cluster isn\u0026rsquo;t just about the PCs; it\u0026rsquo;s about the network connecting them. My nodes kept dropping offline due to network congestion on a single 1Gbps link. Adding dedicated 2.5GbE cards for a private cluster subnet (192.168.69.0/24) made the cluster perfectly stable.\nNow that the physical foundation is laid, it\u0026rsquo;s time to build on top of it. In the next part of this series, I\u0026rsquo;ll dive into the software that brings this hardware to life. We\u0026rsquo;ll cover:\nCreating a ZFS-based NAS: How I\u0026rsquo;m using TrueNAS (or Proxmox\u0026rsquo;s built-in ZFS) to manage my 6TB of storage.\nSpinning up the Kubernetes Cluster: We\u0026rsquo;ll provision the VMs that will form our 3-node k8s cluster using Terraform and Ansible.\nAutomating Everything with GitOps: Finally, I\u0026rsquo;ll show you how I use ArgoCD to automatically deploy and manage all my applications (Jellyfin, Frigate, and more) on Kubernetes.\nStay tuned!\n","date":"28 October 2025","externalUrl":null,"permalink":"/posts/on-premise-hardware/","section":"Posts","summary":"You might have heard about the recent AWS outage that caused many services to go down. To think that half of the internet relies on a single service is crazy, even if it might be the best in its field. The engineers at AWS have done many things to prevent this, such as high availability, multi-region, zones, etc., but if a critical part of it still goes down, the whole thing goes down too. Therefore, if you don’t want to give all your money and trust to a single company, you might want to explore the on-premise option, which is self-hosting.\n","title":"On-Premise 101 (Part 1): Building a 3-Node Cluster","type":"posts"},{"content":"A long time ago, my house only had 1 router, located on the third floor. So whenever we needed to access the internet while having dinner on the first floor, we needed to enable 4G or walk near the stairs just to receive a fraction of the WiFi signal. My father decided to give me an important task, which was to figure out how to solve this issue. I initially bought a simple WiFi extender located on the second floor, but after using it for a month, even though we could receive a strong signal from it, we couldn\u0026rsquo;t access the internet because the signal from the router on the 3rd floor to the WiFi extender on the 2nd floor was still really weak.\nThat\u0026rsquo;s when I decided to ask my dad to invest in a powerful second router located in my room on the 2nd floor, that could connect to the master router via LAN cable and advertise a strong WiFi signal. Researching multiple options, I opted for the Xiaomi CR6608 as it had impressive specs (WiFi 6, 4 antennas, gigabit) and most importantly, it could install OpenWRT. This opened a whole new world for me, because after 3 years, it has taught me many things about networking and the internet, such as subnets, IPv4, DHCP, DNS, and VPNs.\nThis post is a tour of my home network, built on that OpenWRT router. I\u0026rsquo;ll walk through how I designed my network layout, solved DNS and ad-blocking, and built a secure, globally-accessible lab using Tailscale and a cheap VPS.\nPlanning the Network # Currently, this router is configured as a cascaded router behind another router (double NAT). So it is also a DHCP server, which subnet is 10.69.0.0/16, to give IPs to the client devices. It has the total capacity of giving 256*256=65536 IPs:\nPhysical devices (fixed): 10.69.0.1 to 10.69.0.255 (255 ips) Proxmox VMs: 10.69.1.1 to 10.69.1.255 (255 ips) Kubernetes load balancers: 10.69.2.1 to 10.69.2.255 (255 ips) Other WLAN client (dynamic): 10.69.254.1 to 10.69.255.255 (510 ips) I also plan to learn how to set up VLANs for additional security and a separate environment, but that requires a managed switch, which is kinda expensive. For now, it has done an awesome job of being a router, giving IPs and helping the clients connect to the internet. But it also needs to provide DNS for the client devices too.\nBlocking Ads and Local DNS # On the DNS side, I initially used an AdGuard local server hosted directly on the Xiaomi CR6608 to provide DNS for clients. But after running it for a while for my family, I found out that my device couldn\u0026rsquo;t hold up long when multiple devices trying to connect; it kept restarting after experiencing too much load. And also, when I wanted to go out, if I wanted to get the same DNS setup as when I was at home, I needed to configure a lot more to get it working.\nSo in the end, I chose to use NextDNS as my primary DNS solution. With luci-app-nextdns in the OpenWRT packages, my router gets the profile ID for my NextDNS profile and then advertises it so that every client can also have it. What\u0026rsquo;s more, I can still configure my iPhone to have an HTTPS DNS Profile for NextDNS when I go outside.\nAdditionally, it also has the capacity to block ads, parental controls, and blacklist or whitelist any domain you want. This feature is extremely useful if you want to limit yourself from doom-scrolling Facebook, YouTube, or any other services while you\u0026rsquo;re at work, and you can get access back when you\u0026rsquo;re not. It can also provide analytics and logs, which give you more monitoring options.\nAccessing My Network from Anywhere # But using DNS for my local server would not give me immediate access to my server at home. I needed a way to access my local IP while I am outside. Moreover, since my internet modem is also behind a NAT, opening port forwarding would have no effect. After researching, I found these were the options that could help me:\nFeature WireGuard Cloudflare Tunnel Tailscale How it Works Direct Client-to-Server VPN. Your server at home listens for connections. An agent at home sends an outbound connection to Cloudflare. All devices connect to a virtual mesh network, brokered by Tailscale. Setup Difficulty Difficult Medium Very Easy Needs Open Ports? Yes No No Client App Required? Yes (on client devices) No (for web access) Yes (on all devices) Key Pros High performance, self-hosted, full network control. Simple for web access, no client app for browsers, hides your home IP. \u0026ldquo;It just works\u0026rdquo; simplicity, great for CG-NAT, powerful features (subnet routes). Key Cons Hard to set up, requires port forwarding, fails with CG-NAT. Requires a domain, traffic limits (not for video), less flexible for non-web. Requires their app on all devices, relies on a 3rd-party service. Needless to say, I have chosen Tailscale to work with. I couldn\u0026rsquo;t believe it is a free solution because it gives me so much management and many features. The ability to act as both a subnet route (advertise local IP for devices that don\u0026rsquo;t have the app) and an exit node (forward all the traffic through one device) is just super powerful. I did try using Cloudflare Tunnel at the beginning, but it required me to configure every DNS record each time I wanted to access a service, and it also did not allow me to stream video, so I migrated all of it to Tailscale a year ago.\nCurrently, I have Tailscale installed on my OpenWRT router, which advertises a subnet route of 10.69.0.0/16 and also acts as an exit node. If I go outside, I just simply open the Tailscale app on my phone or my laptop, authenticate, and bang, connected. If I carry a device that can\u0026rsquo;t install Tailscale, I can simply connect it to my travel router (which is the gl-ar300m device in the image above), which I have wrote about here.\nDon\u0026rsquo;t get me wrong, Cloudflare Tunnel is still a solid option. It has one feature that Tailscale couldn\u0026rsquo;t provide, which is the ability to allow guest users to connect to my home services without them needing to install an app or authenticate anything. This is extremely useful for a web blog when I want to showcase something. But for now, I have figured out a way to combine Tailscale and an additional VPS (a virtual private server, which is the racknerd-e54f406 device in the image above) to achieve this.\nSecurely Exposing Services to the Public # One day, I was tinkering with the idea of finding the best VPS provider to host some of my services in case my homelab was on maintenance but I still wanted to access critical services such as Bitwarden (a password manager) and Actual-budget (a budgeting service). Then I found this site called RackNerd. They offered an option that shocked me with its price and specs:\n11$ per year 1 vCPU Core 24 GB Pure SSD Storage 1 GB RAM 2 TB Monthly Transfer 1 Gbps Network Port Full Root Admin Access 1 Dedicated IPv4 Address What\u0026rsquo;s more, I realized I could also connect to services at home using Tailscale + Nginx, which is a reverse proxy. With 1 GB RAM, I could easily install Nginx Proxy Manager in a Docker container. I also needed to configure DNS on Cloudflare to route all DNS requests beginning with *.vps.phuchoang.sbs and vps.phuchoang.sbs to match the public static IP of the VPS. The final step was to install Tailscale on it and then type the home IPs on the destination, and I was good to go.\nHowever, I realized the above method was scary because it exposed the services publicly. If an application doesn\u0026rsquo;t have an authentication method or 2FA built-in, I was afraid hackers might attack my services at home. Therefore, I decided to spin up Authelia, an open-source authentication and authorization server. It allowed me to create a side-car authentication service for every \u0026ldquo;proxy host\u0026rdquo; in Nginx Proxy Manager above. All I needed to do was to add myself as a user and adjust the custom Nginx Configuration on the Advanced tab of each \u0026ldquo;proxy host\u0026rdquo;. Now each time you visit, for example nginx.vps.phuchoang.sbs, it will prompt you for a username and password. Even if you get it right, you have to provide the 2FA codes that are set up for each user. Pretty secure, right?\nSummary # It\u0026rsquo;s amazing to think this all started with a simple WiFi problem on the first floor. What began as a need for a second router (the Xiaomi CR6608) quickly became the \u0026ldquo;brain\u0026rdquo; of my entire home network.\nBy leveraging the power of OpenWRT, I built a custom network from the ground up. NextDNS handles clean and secure DNS (plus ad-blocking) for my whole family, both at home and on the go. Tailscale completely solves the challenge of being behind my ISP\u0026rsquo;s NAT, creating a secure mesh network that lets me access my 10.69.0.0/16 subnet from anywhere.\nFinally, adding a cheap RackNerd VPS with Nginx Proxy Manager and Authelia gives me the best of both worlds: a way to securely expose public services (like a blog) or private services (like Actual-budget) to the internet, all while being protected by a strong 2FA authentication portal. This setup is the backbone of all my projects and has been a powerful, hands-on learning experience.\n","date":"27 October 2025","externalUrl":null,"permalink":"/posts/self-hosted-network-design/","section":"Posts","summary":"A long time ago, my house only had 1 router, located on the third floor. So whenever we needed to access the internet while having dinner on the first floor, we needed to enable 4G or walk near the stairs just to receive a fraction of the WiFi signal. My father decided to give me an important task, which was to figure out how to solve this issue. I initially bought a simple WiFi extender located on the second floor, but after using it for a month, even though we could receive a strong signal from it, we couldn’t access the internet because the signal from the router on the 3rd floor to the WiFi extender on the 2nd floor was still really weak.\n","title":"Building a Secure Network for Homelab with OpenWRT, NextDNS, Tailscale, and a VPS","type":"posts"},{"content":"","date":"27 October 2025","externalUrl":null,"permalink":"/tags/network/dns/","section":"Tags","summary":"","title":"network/dns","type":"tags"},{"content":"","date":"27 October 2025","externalUrl":null,"permalink":"/tags/network/self-hosted/openwrt/","section":"Tags","summary":"","title":"network/self-hosted/openwrt","type":"tags"},{"content":"A week ago, my laptop died. When I booted it up, it said something like, fTPM is corrupted, press Y to reset or N to do nothing. Neither of them helped me boot into my machine. So I had a severe mental attack back then, because that machine was the primary one that helped me do my work for school, my lab, and my personal projects. I was lucky to have my friend lend me an old laptop of his, but it was pretty bulky, so I figured I couldn\u0026rsquo;t bring it to school. Therefore, I had to go to the school library to do my work on its public computers.\nLuckily, I have a Microsoft Edge sync account, so all of my tabs, history, extensions, and passwords were replicated exactly like my old setup. However, as it is a school computer, it\u0026rsquo;s managed by the organization, so I couldn\u0026rsquo;t install any programs, [[replicate an exact programming experience on any machine]], or browse my personal home network. Lucky me again, there was a VirtualBox program on the school computer, so I could install Arch on it and run the script to download and configure all of my dotfiles and programs that I frequently use.\nHowever, there was still one big problem: I couldn\u0026rsquo;t access the resources that I host with my homelab, so I still couldn\u0026rsquo;t 100% replicate the network side of my home laptop. Previously, even if I was at school, I could connect via Tailscale to my home router running OpenWRT that advertises a subnet route of 10.69.0.0/16, so I could still access those addresses without having to physically be there. Because the public computer can\u0026rsquo;t install any programs, it can\u0026rsquo;t install Tailscale. But I saw there was a LAN cable connected to it, so I thought maybe I could buy some old router that could also run OpenWRT to hijack the school\u0026rsquo;s internet. Enter the GL.iNet Travel Router.\nThis post is the story of how I turned a tiny, $10 travel router into a powerful networking tool that gave me full, secure access to my entire homelab from a locked-down public computer.\nThe plan # After a long consideration and research, I settled on the GL-AR300M16. It was a tiny, cute little router that was actually (not so much) powerful. It\u0026rsquo;s specs were:\n100 Mbps Ethernet ports 16MB NOR ROM flash (The problem child) 128MB RAM (The lifesaver) Two configurable WAN/LAN ports My goal was to \u0026ldquo;insert\u0026rdquo; my own router between the school\u0026rsquo;s network and the public computer. The computer wouldn\u0026rsquo;t know the difference, but all its traffic would first go through my device, which would be running Tailscale. That way, I could fool the school into thinking I was connected via the school network, but I was actually routing all my packets to my home router. Here\u0026rsquo;s the data flow:\nThe price was really affordable too, only 33 bucks. But as a cheap-ass kid, I was looking to buy something old and used on eBay. Even more surprising, I actually found one that was new/in an unopened box and listed for bidding at 5 bucks. After staying up all night to watch the bid, I successfully got it for 10 + 8 bucks for shipping.\nFirmware reinstallation # After I received the package, the first thing I wanted to do was to uninstall the stock firmware and install the original OpenWRT to free up some space. I don\u0026rsquo;t hold any hatred toward the stock firmware, but with the router only having 16MB of ROM storage, and the firmware size being 15 MB, that left me with less than 1MB to do anything. So, no.\nSearching the internet, I found this site to download the clean version firmware of the router. But when I installed it, it only had OpenWRT version 17, which was so outdated that I couldn\u0026rsquo;t even update the packages properly because the SSL was no longer working. So I hit the internet again to find a guide and, luckily again, I even found the original OpenWRT guide dedicated to this machine.\nThe suggested way of installing OpenWrt on the GL-AR300M is using the u-boot bootloader. It can be accessed by holding the reset button, powering the device and waiting for 5 flashes of the LED. Instructions on how to access u-boot can also be found in the debricking instructions: https://docs.gl-inet.com/en/3/tutorials/debrick/.\nThen one can release the button, and access u-boot on http://192.168.1.1 which requires a PC with a static IP of 192.168.1.2 and netmask 255.255.255.0 .\nAssuming your u-boot is working correctly:\nboot into u-boot (debrick mode) by holding the reset button, powering the device and waiting for the led to flash 5 times. Browse to [http://192.168.1.1](http://192.168.1.1/ \u0026quot;http://192.168.1.1\u0026quot;) Upload glinet_gl-ar300m-nand-initramfs-kernel.bin file to router Wait for it to reboot Telnet to 192.168.1.1 and set a root password, or browse to http://192.168.1.1 if LuCI is installed. You will notice that the router is running in safemode. In order to leave it, one must now flash a regular image factory or sysupgrade (e.g. glinet_gl-ar300m-nand-squashfs-factory.img) from shell or luci. After this second flash, the router will boot normally.\nThe only thing I needed to adjust to access u-boot was that I needed to unplug all the external antennas and the WAN port, leaving only the LAN cable connected to it. Also, I needed to remember to download two files: one initramfs-kernel.bin file for the initial u-boot installation, and the other squashfs-sysupgrade.bin for the sysupgrade installation afterward.\nAlso, as I\u0026rsquo;ll explain later in this article, I was stuck with the issue of Tailscale not working without iptables because OpenWRT decided to use nftables in OpenWRT 22.04 and later. So I needed to install the 21.02 version. Here are the links to the two firmware files:\nopenwrt-21.02.0-ath79-generic-glinet_gl-ar300m16-initramfs-kernel.bin (for the initial u-boot flash) openwrt-21.02.0-ath79-generic-glinet_gl-ar300m16-squashfs-sysupgrade.bin (for the real installation afterward). Install Tailscale # Now for the fun part. I found and tried using this repo but realized the device only had 16MB of NOR ROM, of which 6MB had already been reserved for the firmware. So when I ran the script, it warned me I didn\u0026rsquo;t have enough storage, as it required 15MB.\nTherefore, I decided to follow this repo. Instead of relying on the 16MB NOR ROM storage, it stored the tailscaled and tailscale binaries in /tmp (i.e., the 128 MB RAM) instead. But it still stored the init daemon (which is used for downloading those binaries every time the router boots up) and the state file to keep the login and authentication persistent.\nHowever, with that said, I still had to downgrade the Tailscale version and remove the \u0026ldquo;fetch latest release\u0026rdquo; logic in the usr/bin/tailscale and usr/bin/tailscaled files because the latest version also didn\u0026rsquo;t fit in the available RAM.\ntailscale_version=\u0026#34;1.36.1\u0026#34; # latest_version=`wget -O- https://pkgs.tailscale.com/stable/ | grep tailscale_ | head -1 | cut -d\u0026#39;_\u0026#39; -f 2` # if [ \u0026#34;$tailscale_version\u0026#34; != \u0026#34;$latest_version\u0026#34; ]; then # tailscale_version=$latest_version #fi For Tailscale versions before 1.58.2-1, the init script may need to be modified to force Tailscale to assign an IP to the tailscale0 interface. So I had to modify the init daemon at /etc/init.d/tailscale. After the last procd_append_param, I added: procd_append_param command --tun tailscale0\nAs mentioned previously, Tailscale has trouble running on OpenWRT version 22.04 and later. I tried to follow this guide to fix it, but no luck, so I decided to reinstall firmware 21.02 again.\nwgengine.NewUserspaceEngine(tun \u0026#34;tailscale0\u0026#34;) error: router.Up: setting netfilter mode: exec: \u0026#34;iptables\u0026#34;: executable file not found in $PATH flushing log. logger closing down getLocalBackend error: createEngine: router.Up: setting netfilter mode: exec: \u0026#34;iptables\u0026#34;: executable file not found in $PATH After the re-installation and following the exact guide above, I was finally able to authenticate with Tailscale on the router.\nroot@OpenWrt:~# /etc/init.d/tailscale start root@OpenWrt:~# tailscale up To authenticate, visit: https://login.tailscale.com/a/d7ec72e01e7d4 Success. Some peers are advertising routes but --accept-routes is false # Reboot the machine root@OpenWrt:~# tailscale status Downloading Tailscale 1.36.1_mips .. Downloading \u0026#39;https://pkgs.tailscale.com/stable/tailscale_1.36.1_mips.tgz\u0026#39; Connecting to 199.38.181.239:443 Writing to stdout tailscale_1.36.1_mips/tailscale - 100% |*******************************| 20505k 0:00:00 ETA Download completed (20997485 bytes) Done! 100.98.192.109 openwrt-1 mrphuc26032006@ linux - 100.121.251.90 archlinux mrphuc26032006@ linux offline 100.72.74.109 ipad161 mrphuc26032006@ iOS offline 100.111.147.16 iphone-13 mrphuc26032006@ iOS offline 100.110.49.37 openwrt mrphuc26032006@ linux idle; offers exit node 100.68.207.111 racknerd-e54f406 mrphuc26032006@ linux idle; offers exit node The next step was to route all the traffic that connects to this router to my home network. This was easily done by following this guide.\nCreate a new unmanaged interface via LuCI: Network → Interfaces → Add new interface\nName: tailscale Protocol: Unmanaged Device: tailscale0 Create a new firewall zone via LuCI: Network → Firewall → Zones → Add\nName: tailscale Input: ACCEPT (default) Output: ACCEPT (default) Forward: ACCEPT Masquerading: on MSS Clamping: on Covered networks: tailscale Allow forward to destination zones: Select your LAN (and/or other internal zones or WAN if you plan on using this device as an exit node) Allow forward from source zones: Select your LAN (and/or other internal zones or leave it blank if you do not want to route LAN traffic to other tailscale hosts) Click Save \u0026amp; Apply\n(Optional) Connect to WPA2-EAP network such as eduroam # Although my router has 2 WAN/LAN ports, it also has the capability to act as a WiFi repeater. This can be extremely useful if I\u0026rsquo;m wandering around the school and can\u0026rsquo;t find any LAN ports on the wall; I can still connect to the eduroam network and advertise it back to my phone or tablet to access my home network.\nHowever, eduroam is kinda tricky to set up, even on a computer client. Lucky me again, I found this guide to help me set it up. Albeit, I had to install the wpad-wolfssl package instead of wpad, since I was using wolfssl as a backend service. The normal wpad package didn\u0026rsquo;t allow me to verify the SSL certificate after connecting to it.\n\u0026ldquo;eduroam\u0026rdquo; is a WWPA2-EAP network that allows members of higher education and other institution around the world to use each others WiFi networks with their home credentials. As such, the setup is slightly more complicated than that of other WiFi clients. Especially, the wpad package needs to be upgraded:\nInstall the full version of wpad: opkg update; opkg remove wpad-mini; opkg remove wpad-basic; opkg install wpad; reboot (via SSH, but the web UI works as well). Click \u0026ldquo;Scan\u0026rdquo; on either WiFi interface (but doing this for both seems to create problems with DHCP client) on http://192.168.8.1/cgi-bin/luci/admin/network/wireless, select \u0026ldquo;Join network\u0026rdquo; for any \u0026ldquo;eduroam\u0026rdquo; connection/AP. Select the few possible settings as appropriate, enter anything as password for now, \u0026ldquo;submit\u0026rdquo;. Under \u0026ldquo;Wireless Security\u0026rdquo;, select WWPA2-EAP as \u0026ldquo;Encryption\u0026rdquo;, and set everything else according to your institutions eduroam configuration. Here\u0026rsquo;s my school\u0026rsquo;s configuration: (Optional) Install NextDNS to have local DNS resolution and Ads Blocking on the go # If you want more detail on how I set up NextDNS and Tailscale, you can read it here. But for now, it is my main DNS server to resolve my Kubernetes Traefik resolutions, for ad-blocking services, and for parental controls to limit myself from doom-scrolling too much. Installing it was easier than Tailscale. Just get the NextDNS configuration profile ID, paste it in after you install the package, and you\u0026rsquo;re good to go.\nGo to System -\u0026gt; Software Click the \u0026ldquo;Update lists\u0026rdquo; button Enter \u0026ldquo;luci-app-nextdns\u0026rdquo; in the \u0026ldquo;Download and install package\u0026rdquo; field and click \u0026ldquo;OK\u0026rdquo; Go to Services -\u0026gt; NextDNS and configure NextDNS Summary # For $18, even though it\u0026rsquo;s a weak router, I couldn\u0026rsquo;t ask more of it. It can run Tailscale, act as a WiFi repeater, and handle NextDNS resolution, and with all of that, it still has 28MB left in RAM. What\u0026rsquo;s not to like? Definitely a lifesaver at a time when my laptop was broken. Maybe with this, I will begin to prefer going to school to study and code more than staying at home.\n","date":"20 October 2025","externalUrl":null,"permalink":"/posts/self-hosted-travel-router/","section":"Posts","summary":"A week ago, my laptop died. When I booted it up, it said something like, fTPM is corrupted, press Y to reset or N to do nothing. Neither of them helped me boot into my machine. So I had a severe mental attack back then, because that machine was the primary one that helped me do my work for school, my lab, and my personal projects. I was lucky to have my friend lend me an old laptop of his, but it was pretty bulky, so I figured I couldn’t bring it to school. Therefore, I had to go to the school library to do my work on its public computers.\n","title":"Accessing a Homelab from a Locked-Down PC with a Travel Router","type":"posts"},{"content":"","date":"31 May 2025","externalUrl":"https://github.com/phuchoang2603/kubernetes-proxmox","permalink":"/projects/kubernetes-proxmox/","section":"Projects","summary":"This project automates the provisioning and configuration of a RKE2 Kubernetes cluster on Proxmox using Terraform and Ansible.","title":"Automate provisioning Kubernetes cluster on Proxmox with Terraform + Ansible","type":"projects"},{"content":" How I got here # Hey! I\u0026rsquo;m Hoang Xuan Phuc, but everyone just calls me Felix.\nI\u0026rsquo;m a CompSci student at the University of South Florida (USF), hailing from Vietnam. Growing up, my life was basically a script: Wake up at 6 a.m., school at 7, extra school at 5, eat, sleep, repeat. That was my loop until 5th grade, when my mom gifted me an old laptop.\nMy first thought? Gaming on it all day long. But that dream died fast. I quickly realized I had inherited a \u0026ldquo;potato laptop\u0026rdquo; after trying to run some random first-person shooter game (you know, back when they still used Adobe Flash Player). Sure, it was playable, but the lag was so bad I\u0026rsquo;d have to be a psychic to get a shot in before getting blasted.\nBeing a problem-solving nerd, I got hooked on a new question: How do I make this thing not suck?\nSo, thanks to Google and QuanTriMang.com, I went down the rabbit hole of 20-step optimization guides. I was editing startup programs in msconfig.exe, killing services in services.msc, and even foolishly trying to \u0026ldquo;download more RAM\u0026rdquo; by messing with pagefile.sys (protip: don\u0026rsquo;t do this). Honestly, every megabyte of RAM I freed up was a bigger rush than actually playing the game.\nThis tinkering addiction escalated. Before I knew it, I\u0026rsquo;d installed Linux, ditching games entirely for the promise of it running faster. And man, the learning curve wasn\u0026rsquo;t a curve, it was a cliff. I spent more time fixing the OS than actually using it.\nTouch-pad scrolls too fast? Have fun digging in random config files in the /etc directory; you need to manually change the file content, because there isn\u0026rsquo;t any proper GUI to fix this. How to change it? Oh, the internet said I should use the vim program to edit the file. Okay, it opened, but why can\u0026rsquo;t I type anything? You have to press i to enter Insert Mode? Okay, copied and pasted my line there\u0026hellip; now how the hell do I save and quit??? Yeah, it was annoying. But it was also so much fun. I realized I love digging into the guts of a computer and making it my own. Now, my life script is: wake up at 12 am, school till 6, eat, fake sleep, and then run random commands till 4am, repeat. And then, later on, I found myself not just playing around with my own laptop but playing around with other things like blocking ads on the router, self-hosted a Netflix and MangaDex at home, running Switch emulation on an iPad, hacking a PS4 to install unlimited games, \u0026hellip;\nThe problem is, my brain isn\u0026rsquo;t big enough to remember all these commands and weird tricks, and they\u0026rsquo;re all so cool I just want to talk about it all day long. So that\u0026rsquo;s where this blog comes in. Think of it as my public brain-dump - a place to store my \u0026lsquo;hidden recipes\u0026rsquo; and all the cool (or weird) things I find. It\u0026rsquo;s my personal notebook, a record of my journey, and maybe, just maybe, it\u0026rsquo;ll help or inspire you to break\u0026hellip; er, I mean, tinker with your own computer, too. :wq\nMy \u0026ldquo;On-Paper\u0026rdquo; Stats # Alright, enough stories. If you\u0026rsquo;re interested in my professional journey, my skills, and all that \u0026ldquo;hire me\u0026rdquo; stuff, you can find the details in the resume I\u0026rsquo;ve embedded below.\n","date":"4 April 2025","externalUrl":null,"permalink":"/about/","section":"/home/fel1x","summary":"How I got here # Hey! I’m Hoang Xuan Phuc, but everyone just calls me Felix.\n","title":"Welcome to my blog!","type":"page"},{"content":"If you thought the biggest gaming news was the upcoming Switch 2, think again. The emulation community has made a breakthrough: you can now run Nintendo Switch games on your iPhone or iPad at buttery-smooth 60FPS—all offline, with no jailbreak, no shady VPNs, and zero sketchiness.\nBackground # What is JIT? Why is it important for emulation? # Just-In-Time (JIT) compilation is a method used by emulators to dramatically improve performance by converting code into machine instructions while the program is running. Normally, an emulator has to interpret every single instruction one-by-one, which is slow. JIT speeds things up by compiling larger chunks of code in real time, allowing games to run closer to native speeds. However, Apple has strict security policies that prevent unauthorized JIT execution on iOS devices. There are some workarounds like JITStreamer or SideJITServer but they all require an internet connection. StikJIT and StosVPN now bypass this restriction by enabling JIT locally, without external servers, making offline Nintendo Switch emulation possible on iPhones. Here\u0026rsquo;s the difference between no JIT vs JIT-enabled when running emulator on iOS.\nPPSSPP On iOS 17 (NO JIT vs JIT) iPhone XR\nStosVPN – Internal VPN for enabling JIT without internet # Unlike commercial VPNs that often collect user data, StosVPN simply creates a virtual VPN on the device itself, tricking the system into thinking a “trusted connection” is being made. This lets iOS enable JIT without needing an actual internet connection—boosting privacy and eliminating security risks.\nThis is a breakthrough that allows Switch emulators to run fully offline with high performance.\n🔐 No internet connection needed – No data sent out – No unusual battery drain.\nEssential Tips Before You Begin # Recommended hardware - IOS 17+, Iphone/Ipad with more than 4 GB RAM. Disable Low Power Mode – Emulation requires full performance. If iOS throttles the CPU, the game will lag. Don’t close important background apps – Some apps like SideStore and LiveContainer must keep running for stable emulation. Prepare a Windows or macOS computer. You’ll need one to set up SideStore initially and to generate a pairing file. It\u0026rsquo;s best to put all the needed apps into one folder. Use that folder to transfer games – You can create a Windows shared folder and mount it in the iOS Files app for easier game management. Installing SideStore # SideStore is a sideloading tool (lets you install apps outside the App Store) that doesn’t require jailbreaking and is regularly updated by the community.\nInstallation steps:\nRead the official guide for more in-depth tutorials: SideStore Docs\nGenerate a pairing file:\nDownload jitterbugpair.exe Run it and transfer the *.mobiledevicepairing file to your phone Install SideStore:\nInstall AltServer Windows Guide Hold Shift + Click the AltServer tray icon and sideload the SideStore IPA Final Tweaks:\nInstall StosVPN from the App Store and enable it Open SideStore, pair it with the pairing file Refresh the app and remove previous AltServer certificates Why we need to refresh Sidestore (you may ask)? Normally, Apple limits sideloaded apps (using a free developer account) to 7 days of usage before you have to “refresh” (i.e., re-sign the IPA with a new certificate). When you install SideStore using AltServer, it stops using AltServer’s certificate and instead creates its own certificate, managed within the app itself. This means you won’t need a computer later—just open SideStore on your phone and tap “Refresh” to handle everything automatically. To do that, SideStore needs a way to simulate a local server environment to trick iOS into thinking the app signing process is legitimate. The trick here is using an internal VPN (StosVPN), which creates a loopback server directly on the device.\nInstalling LiveContainer # Apple allows only 3 sideloaded apps to run at once. LiveContainer bypasses this by running apps “inside” it, like a virtual machine. Here’s how to install it:\nDownload the LiveContainer IPA from HugeBlack’s fork Actions tab (requires GitHub account) Open SideStore with StosVPN enabled and add LiveContainer IPA. In LiveContainer Settings: Tap \u0026ldquo;Patch SideStore/AltStore\u0026rdquo; to reinstall it with tweaks. After installation: Reopen SideStore/AltStore. Return to LiveContainer: Tap \u0026ldquo;Test JIT-Less Mode\u0026rdquo;—if it says \u0026ldquo;Test Passed,\u0026rdquo; you’re good to go. Install a second instance of LiveContainer via the main LiveContainer app. In LiveContainer Settings: Set JIT Enabler to StikJIT (Another LiveContainer). Installing MeloNX \u0026amp; Enable StikJIT \u0026amp; Increasing RAM Limits # Apple limits apps to using only half of the device’s RAM, but GetMoreMemory by HugeBlack bypasses this restriction.\nDownload MeloNX \u0026amp; memory entitlement: MeloNX Repo and StikKIT IPA from StikJIT GitHub.\nAdd all three apps to the Apps section in LiveContainer. For each app: long-press the icon → Settings → Convert to Shared App.\nEnable file picker \u0026amp; local notifications in MeloNX settings. Run memory entitlement and log into your account to enable the entitlement for:\nLiveContainer LiveContainer2 MeloNX If errors occur, clean up Keychain and try again. Reinstall LiveContainer \u0026amp; LiveContainer2 to apply the configuration.\nReinstall SideStore from the app (do not just refresh it).\nUpload the pairing file in StikJIT on LiveContainer2 and enable \u0026ldquo;Auto Quit After Enabling JIT.\u0026rdquo; Run MeloNX via LiveContainer1.\nAdding keys, firmware, games on MeloNX # First-time setup:\nLaunch MeloNX via LiveContainer Choose your prod.keys and title.keys files Select your Switch firmware .zip Go to settings and check to see if it has JIT and extended RAM enabled You\u0026rsquo;re done! Just tap MeloNX from LiveContainer whenever you want to play, even offline. Add games (.NSP or .XCI) by tapping the ➕ button.\nI won’t go into detail on how to acquired keys, firmware, and games since this involves piracy. As far as I know, aside from downloading illegally, you can extract your own keys and firmware from your own Switch. For, uh, testing purposes, here’s a little base64:\nS2V5cyAmIEZpcm13YXJlOiBodHRwczovL3Byb2RrZXlzLm5ldC8NCkdhbWVzOiBodHRwczovL25zd2dhbWUuY29tLw== Some useful tips I’ve come across:\nAlways download both the game and its update file for the best performance. Large files may get stuck during transfers due to MeloNX’s lack of a proper file transfer UI. MeloNX Settings for best performance or least ram usage # Use the following settings to get the best possible performance # Shader Cache: On (may causes games to use much more ram which can cause crashes on devices with not enough ram tho) Disable VSync: Off (Enable if you want more than 30/60fps if your device can handle it) Texture Recompression: On MacroHLE: On Docked Mode: Off Resolution Scale: Use the lowest resolution you still find good where it doesn’t crash Memory Manager: Sometimes \u0026#34;Host Unchecked (fast, unstable / unsafe\u0026#34; or \u0026#34;Host (fast)\u0026#34; has better performance Ignore Missing Services: On Debug Logs: Off Trace Logs: Off MVK: Pre-fill Metal Command Buffers: Off Use the following settings to get the least amount of ram usage in a game # Shader Cache: Off Texture Recompression: On MacroHLE: On Docked Mode: Off Resolution Scale: The lower, the better Expand Guest Ram: Off Ignore Missing Services: On Debug Logs: Off Trace Logs: Off Memory Manager Mode: Host (fast) Disable PTC: On List of games that MeloNX currently supports # Compatibility | MeloNX # 3gb+ Devices # Minecraft: Nintendo Switch Edition (Bedrock requires a 8gb+ ram) Sonic Mania Captain Toad: Treasure Tracker Sniper Rescue Helltaker (Homebrew) VVVVVV Cheez it the game One shot world machine 4gb+ Devices (May work on 3gb devices if on ios 18.2.x) # Hue (probably 3gb devices too) Celeste Mario vs Donkey Kong (requires 2-3 tries the first time) Undertale (probably 3gb devices too) Star Wars: The Force Unleashed Carrion Trombone Champ (requires gyro controls which aren’t implemented yet) Dead Cells Thumper Farming Simulator 20 Super Meat Boy Nintendo Switch Online(All) Devil May Cry 3: Special Edition 6gb+ Devices (May work on 4gb devices if on ios 18.2 - 18.2.x) # Untitled Goose Game SCHiM Oceanhorn Super Mario Maker 2 Unravel 2 Super Mario Bros. Wonder Sonic Superstars Cult of Lamb Super Mario 3D World Portal ANTONBLAST Mario Kart 8 Deluxe Super Mario Bros. U Deluxe Outlast Links awakening Pokémon Legends Arceus PayDay 2 Pokémon Sword Oceanhorn (Some flickering but not unplayable) Farming Simulator 23 Goat Simulator Super Mario 3D World Animal Crossing: New Horizons Diablo 3 Eternal Edition Persona 5 Royal Persona 4 Golden Ni No Kuni 2 8gb+ devices (May work on 6gb devices if on ios 18.2.x) # Outer Wilds (set CPU Mode to Software) Skyrim Super Mario Odyssey Cuphead Arkham City Call of Juarez: Gunslinger Thief Simulator Asterix \u0026amp; Obelix XXL 3 Super Mario Party: Superstars Mario \u0026amp; Sonic: At The Olympic Games Super Mario 3D All-Stars (very slow performance) Need for Speed: Hot Pursuit Breath of the Wild Minecraft (Bedrock, Legacy works down to 3gb devices) Burnout Paradise Remastered Echoes of Wisdom Super Mario RPG Splatoon 2 Mario Rabbids: Kingdom Battle Ori and the Blind Forest Splatoon 3 Xenoblade Chronicles 3 Xenoblade Chronicles: Definite Edition 16gb iPads (also works on 8gb devices if on iOS 18.2.x) # Kirby and the Forgotten Land Lego Starwars: The Skywalker Saga (takes a few tries) Tears of the Kingdom Mario Tennis Aces Games which don’t boot/Boots then crash: # Stray Red Dead Redemption (works on Pomelo) Hogwarts Legacy The Witcher 3 Yoshi’s Crafted World DOOM Eternal Terraria Pikuniku (works on Pomelo) Mario Strikers No man’s sky Wolfenstein 2 Outlast 2 Dying Light Boots but too many graphical glitches # Octopath Travaler 2 Credits \u0026amp; Sources # This guide was compiled from various sources and contributions:\nHugeBlack: Developer of LiveContainer and GetMoreMemory 0-Blu: Developer of StikJIT MeloNX Team: Creators of the MeloNX emulator. You can also join the community here. SideStore Team: For enabling sideloading on iOS Various GitHub Repositories \u0026amp; Documentation: SideStore Docs LiveContainer GitHub StikJIT GitHub MeloNX Repo r/EmulationOniOS community, especially this post How to install MeloNX and get it working with fully offline JIT activation. A step by step guide. : r/EmulationOniOS Final Thoughts # Running Tears of the Kingdom on an iPhone sounds like science fiction—but it’s real. Once it’s all set up, you’ll be gaming at full speed, offline, on hardware that was should be meant for this. Even crazier? This lays the groundwork for full-blown VM emulation. Some users are already running macOS Sonoma on iPad using UTM.\n","date":"3 April 2025","externalUrl":null,"permalink":"/posts/emulation-switch-ipad/","section":"Posts","summary":"If you thought the biggest gaming news was the upcoming Switch 2, think again. The emulation community has made a breakthrough: you can now run Nintendo Switch games on your iPhone or iPad at buttery-smooth 60FPS—all offline, with no jailbreak, no shady VPNs, and zero sketchiness.\n","title":"Running Nintendo Switch Games on iPhone at 60FPS – Free, No Jailbreak Needed","type":"posts"},{"content":" Video demo: # The Struggle: Why I Needed This # Have you ever stumbled upon a manga so niche and obscure that it never gets an official English release? That was me—desperately trying to read intimate and underrated series that weren’t even on MangaDex, likely due to copyright issues.\nAt first, I took the long road: hunting for raw Japanese scans, screenshotting pages, and running them through image translation tools. It worked\u0026hellip; barely. The process was painfully slow and completely broke my immersion. Instead of enjoying the story, I spent more time waiting on awkward machine translations. It was, in a word, disruptive.\nThen it hit me: I already had Suwayomi, a powerful manga server that supports multiple sources and local file management. Why not build my own translation pipeline?\nFinding a Better Way # My first attempt was to look for an existing solution. I found a closed-source iOS app that seemed promising, but when I saw their $15/month paywall, I immediately bailed. (In case you\u0026rsquo;re curious, here\u0026rsquo;s the link.)\nThat led me to GitHub, where I stumbled upon this amazing project. The demo results blew me away, and I knew I had to dive in and make it work for my setup.\nPrerequisite # To follow along, you’ll need Suwayomi Server installed. You can check out the official Docker setup here:\nSuwayomi-Server-docker GitHub\nAfter installation, set up the download folder and local source location. Reference:\nSuwayomi Local Source Wiki\nHere’s my folder structure for Suwayomi:\n. ├── data #suwayomi folder │ ├── downloads # default download folder │ │ ├── mangas │ │ │ └── Rawkuma (JA) │ │ │ ├── Batsu Hare │ │ │ │ ├── Chapter 100 ... │ │ │ ├── Grapara! │ │ │ │ ├── Chapter 69 ... │ │ │ ├── Guilty Circle │ │ │ │ ├── Chapter 1 ... │ │ │ ├── Isekai Saikouhou no Guild Leader │ │ │ │ ├── Chapter 1 ... │ │ │ └── Spy X Family │ │ │ ├── Chapter 105.5 ... │ │ └── thumbnails │ ├── translated # local source location │ │ ├── Batsu Hare │ │ │ ├── Chapter 100 ... │ │ ├── Grapara! │ │ │ ├── Chapter 69 ... │ │ ├── Guilty Circle │ │ │ ├── Chapter 1 ... │ │ ├── Isekai Saikouhou no Guild Leader │ │ │ ├── Chapter 1 .... │ │ └── Spy X Family │ │ ├── Chapter 105.5 .... └── manga-image-translator ├── * ... ├── batch-script.py # additional batch script file that I wrote to automate the translation. The First Steps: Getting the Basics Running # The manga-image-translator repo seems pretty solid at first try. It provide me the default script that can translate multiple chapters at once.\nScan the INPUT_FOLDER (.i.e. the downloads folder I set up above) for new chapters. Run the manga translator command. Save the output in a matching OUTPUT_FOLDER. (.i.e the translated folder) Example set up:\nsudo apt install cython3 conda create -n manga-trans python=3.12 pip python -m manga_translator local -v -i ../suwayomi/data/downloads/mangas/Rawkuma\\ \\(JA\\)/Grapara\\!\\ Raw/Chapter\\ 13/ --output ../suwayomi/data/translated/Grapara\\!\\ Raw\\ translated/Chapter\\ 13/ --use-gpu --config-file examples/config-example.json This worked, but there was one big problem: it wasn’t fully automated and has a lot of command flags that I need to remember. I had to manually re-run the script every time a new chapter arrived. Clearly, there had to be a better way.\nAutomating the Pipeline: Watching for New Manga Chapters # I started exploring file-watching solutions and tested various Linux tools like inotify and watchdog. However, they didn’t track entire directories the way I needed. I wanted a script that:\nMonitors INPUT_FOLDER (and subfolders) for new content. Triggers translation automatically when a new chapter appears. Maintains the same folder structure in OUTPUT_FOLDER. Enter Python watchdog library - a game-changer. With it, I wrote a script that:\nWatches the manga download directory. Detects when a new chapter folder appears. Extracts the relative path and mirrors it in the output directory. Here’s the script:\nimport os import time import subprocess from watchdog.observers import Observer from watchdog.events import FileSystemEventHandler # ===== Configuration ===== INPUT_ROOT = \u0026#34;data/downloads/mangas/Rawkuma (JA)\u0026#34; OUTPUT_ROOT = \u0026#34;data/translated\u0026#34; CONFIG_FILE = \u0026#34;examples/config-example.json\u0026#34; # ========================== class TranslationHandler(FileSystemEventHandler): def on_created(self, event): if event.is_directory: print(f\u0026#34;Detected new folder: {event.src_path}\u0026#34;) self.process_new_folder(event.src_path) def process_new_folder(self, input_path): # Get relative path from input root relative_path = os.path.relpath(input_path, INPUT_ROOT) dest_path = os.path.join(OUTPUT_ROOT, relative_path) # Create output directory if it doesn\u0026#39;t exist os.makedirs(dest_path, exist_ok=True) command = [ \u0026#34;python\u0026#34;, \u0026#34;-m\u0026#34;, \u0026#34;manga_translator\u0026#34;, \u0026#34;local\u0026#34;, \u0026#34;-v\u0026#34;, \u0026#34;-i\u0026#34;, input_path, \u0026#34;--dest\u0026#34;, dest_path, \u0026#34;--config-file\u0026#34;, CONFIG_FILE, \u0026#34;--use-gpu\u0026#34;, ] try: print(f\u0026#34;Starting translation for: {relative_path}\u0026#34;) subprocess.run(command, check=True) print(f\u0026#34;Successfully translated: {relative_path}\\n\u0026#34;) except subprocess.CalledProcessError as e: print(f\u0026#34;Error translating {relative_path}:\u0026#34;) print(f\u0026#34;The command failed with return code {e.returncode}\u0026#34;) print(\u0026#34;Command executed:\u0026#34;, \u0026#34; \u0026#34;.join(command)) def main(): # Validate paths if not os.path.isdir(INPUT_ROOT): raise ValueError(f\u0026#34;Input folder does not exist: {INPUT_ROOT}\u0026#34;) os.makedirs(OUTPUT_ROOT, exist_ok=True) # Set up folder observer event_handler = TranslationHandler() observer = Observer() observer.schedule(event_handler, INPUT_ROOT, recursive=True) observer.start() try: print(f\u0026#34;Watching directory tree: {INPUT_ROOT}\u0026#34;) print(f\u0026#34;Mirroring structure to: {OUTPUT_ROOT}\u0026#34;) print(\u0026#34;Press Ctrl+C to stop monitoring...\u0026#34;) while True: time.sleep(1) except KeyboardInterrupt: observer.stop() observer.join() if __name__ == \u0026#34;__main__\u0026#34;: main() Running It as a Background Service or a Docker container # I wanted the script to run 24/7 without manually starting it each time. There are two ways to achieve this:\nRun the Python script as a systemd service (requires tinkering with conda). Run it as a Docker container (recommended).\u0026quot;* Option 1: systemd Service (for local installs, not recommended) # Created a new service file: sudo nano /etc/systemd/system/manga-trans.service Added this configuration: [Unit] Description=Manga translation for Suwayomi After=network.target [Service] User=1000 Group=1000 WorkingDirectory=/DATA/AppData/manga-image-translator ExecStart=/bin/bash -c \u0026#34;source ~/miniforge3/etc/profile.d/conda.sh \u0026amp;\u0026amp; conda activate manga-trans \u0026amp;\u0026amp; python batch-script.py\u0026#34; [Install] WantedBy=multi-user.target Enabled and started the service: sudo systemctl daemon-reload sudo systemctl enable manga-trans.service sudo systemctl start manga-trans.service Option 2: Docker Container (recommended) # Rather than relying on conda, I containerized the entire application and push it to my own registry. To run it, you must first create a .env file to specify where the data gonna stored.\nMANGA_FOLDER=/mnt/storage/media/manga APPDATA=/mnt/storage/appdata/suwayomi TZ=America/New_York You also need to change the default location where the new manga will be download to in the Suwayomi web interface.\nHere\u0026rsquo;s my docker-compose.yml setup for the full stack deployment.\nname: suwayomi services: suwayomi: container_name: suwayomi environment: - EXTENSION_REPOS=[\u0026#34;https://raw.githubusercontent.com/keiyoushi/extensions/repo/index.min.json\u0026#34;] - FLARESOLVERR_ENABLED=true - FLARESOLVERR_URL=http://flaresolverr:8191 - TZ=${TZ} hostname: suwayomi image: ghcr.io/suwayomi/suwayomi-server:preview ports: - 4567:4567 restart: always volumes: - ${MANGA_FOLDER}:/home/suwayomi/data - ${APPDATA}/data:/home/suwayomi/.local/share/Tachidesk flaresolverr: container_name: flaresolverr environment: - TZ=${TZ} hostname: flaresolverr image: ghcr.io/flaresolverr/flaresolverr:latest ports: - 8191:8191 restart: unless-stopped manga-image-translator: # build: # context: ${APPDATA}/manga-image-translator image: ghcr.io/phuchoang2603/manga-image-translator:v0.1.0 container_name: manga-image-translator command: batch-script.py volumes: - ${APPDATA}/manga-image-translator/:/app/ - ${APPDATA}/facehuggingcache:/root/.cache/huggingface/ - ${MANGA_FOLDER}:/app/data ipc: host # For GPU deploy: resources: reservations: devices: - capabilities: [gpu] Now, it runs in the background automatically whenever I download a new manga chapter. No more waiting, no more manual intervention—just seamless reading.\nFinal Thoughts # This project completely transformed how I read untranslated manga. No more screenshots, slow translators, or endless searching. Everything is automated, seamless, and runs quietly in the background.\nIf you\u0026rsquo;re someone who digs through obscure manga titles like me, I highly recommend trying this out.\n","date":"3 March 2025","externalUrl":null,"permalink":"/posts/self-hosted-manga-trans/","section":"Posts","summary":"Video demo: # The Struggle: Why I Needed This # Have you ever stumbled upon a manga so niche and obscure that it never gets an official English release? That was me—desperately trying to read intimate and underrated series that weren’t even on MangaDex, likely due to copyright issues.\n","title":"Automating Manga Translations: My Journey to a Seamless Reading Experience","type":"posts"},{"content":"","date":"16 November 2023","externalUrl":null,"permalink":"/series/organizing-my-digital-life/","section":"Series","summary":"","title":"Organizing My Digital Life","type":"series"},{"content":"We all love the idea of being super organized, right? We make these grand plans, give it our all for a few weeks or months, and then, bam! We\u0026rsquo;re back to the good old \u0026ldquo;throw it in the downloads folder with everything else\u0026rdquo; routine. It happens to the best of us, and I\u0026rsquo;m guilty too. But hey, even though I\u0026rsquo;ve still got loads of room for improvement, I can proudly say I\u0026rsquo;ve gotten way faster at finding my files.\nPhase One (2014-2019): The Awakening # I stored all my files in the initial phase in the download folder. At that time, with my first laptop and just one attached hard drive, I wasn\u0026rsquo;t concerned about efficiently organizing my files. Typically, I heavily rely on browsers while using our PCs. Most of my files originated from internet downloads—games, program installation, and office files from Gmail, Facebook, and Messenger. As you may know, none of these were particularly crucial: once I downloaded them, I never looked back. However, my awareness of this inefficient system grew as I began experiencing difficulties locating specific files and noticed duplications due to multiple downloads. The tipping point was when I spent nearly an hour searching for a homework document, resulting in a late submission to my teacher. This prompted me to reconsider and change my approach.\nImage courtesy of Folder Tidy\nPhase Two (2019-2021): Aesthetic Revolution # I initiated using folders to categorize file types and employed Internet Download Manager to organize downloaded files based on filename extensions automatically. As someone who values aesthetics, I went further by customizing folder icons for each category, replacing the default \u0026ldquo;folder\u0026rdquo; icon on Windows. Placing these folders in Quick Access significantly enhanced the efficiency of file retrieval.\nHowever, I found the process inefficient when collaborating online with others for studying or project presentations. Organizing files downloaded from shared Drive folders and re-uploading them became time-consuming. Consequently, I realized the need to alter my file organization methods for both offline and online activities.\nPhase Three (2021-2023): Google Drive Renaissance # I dedicated time to thoroughly reorganizing my Google Drive, which had been messy from its use as temporary storage, before transferring files to my offline storage. Despite long-standing plans to address this, the chaotic state of my Google Drive had deterred me. The turning point came when I had to switch from my laptop to a new PC. In the past, I often neglected the backup process: I would have unplugged my hard drive and connected it to the new PC, risking the loss of essential files. But the thing is, there were many important files on it, and I was told by many sources that I should back it up to several Cloud Storage before executing the change. That also allowed me to rearrange things, e.g., filtering out unnecessary files, moving big files onto the USB, and uploading essential files to Google Drive.\nHere\u0026rsquo;s my Drive after fixing it. The two folders, \u0026ldquo;Application\u0026rdquo; \u0026amp; \u0026ldquo;College,\u0026rdquo; are synced with my offline files\nAdditionally, I discovered the Google Drive for Desktop App. It mirrors my Drive files, mounting them to the chosen location\u0026rsquo;s virtual disk. This means all the changes I made on the offline files will be automatically synced with the files on the cloud, reducing the significant time of waiting for uploading.\nCurrent \u0026amp; Future: Expanding Horizons # Recently, I obtained a 5TB OneDrive from the Microsoft 365 free E5 Developer Pack, expanding my file storage capacity. This includes game installation files, video editing projects, and downloaded online courses. Files for collaboration with friends and teachers, such as homework assignments, IELTS, and SAT papers, are stored on Google Drive. Moreover, with the arrival of a refurbished computer that I turned into a Synology NAS, I now have more options for file storage. This NAS serves as my torrent server for streaming films, shows, audiobooks, etc., through platforms like Jellyfin and Audiobookshelf. I plan to share my journey of building self-hosting in the near future, but for now, I am satisfied with my current setup.\nIn the future, I may adopt new techniques to organize my files more efficiently. As of now, I have switched to macOS, and with the wonderful search app called Raycast (Spotlight alternative), I guess I may return to the era where I would store my files a bit more freely, as opposed to the strict tree structure now. Because with the tree structure method, it still takes me several steps to navigate to the files I want. But with the file search method, I just need to learn how to name the files mnemonic, and I can have my files right in the second.\n","date":"16 November 2023","externalUrl":null,"permalink":"/posts/productitvity-file-management/","section":"Posts","summary":"We all love the idea of being super organized, right? We make these grand plans, give it our all for a few weeks or months, and then, bam! We’re back to the good old “throw it in the downloads folder with everything else” routine. It happens to the best of us, and I’m guilty too. But hey, even though I’ve still got loads of room for improvement, I can proudly say I’ve gotten way faster at finding my files.\n","title":"Organizing My Digital Life - A Nerdy Journey Across Drives, File Management","type":"posts"},{"content":"","date":"16 November 2023","externalUrl":null,"permalink":"/tags/productivity/","section":"Tags","summary":"","title":"productivity","type":"tags"},{"content":"Today, I would like to introduce you to the journey of optimizing my workspace environment, or in simpler terms, my desk setup. In fact, I am not someone who strictly adheres to principles in life. However, I really enjoy working in a disciplined environment and always strive to optimize it for effective studying and working. To achieve the setup I have today, I had to tear down and rebuild this workstation more than 10 times. Finally, I am truly satisfied with it. It not only satisfies me aesthetically but also makes me eager to work as soon as I step into this room (well, not quite true because nowadays most of the time I feel most productive when working from a coffee shop\u0026hellip;).\nInitial set-up from 2021, when I was just a 9th-grade middle schooler # Looking at the picture, you would probably guess that this is an extremely simple setup. It shows a Lenovo 14-inch Intel Celeron old school laptop that I inherited from my mom when I was just a 7th-grader. It\u0026rsquo;s a typical low-end Windows laptop from before 2015, and I couldn\u0026rsquo;t even take it anywhere because the connector between the monitor and the keyboard was broken. I remember trying to install some games on it, but the only one that ran smoothly was Halflife 2, which is from the 2000s. Despite its limitations, I feel positive towards this laptop because it served my educational needs and taught me how to \u0026ldquo;optimize\u0026rdquo; things\u0026hellip;\nOver the years, although there were countless times when this low-end laptop frustrated me with its 2GB RAM and inability to open more than three Google Chrome tabs, it still fulfilled my educational requirements. Adding a second monitor that my mom had left over from her office was a game changer for me, especially during the COVID-19 pandemic when I had to attend Zoom classes. This additional monitor played a crucial role in helping me get accepted into three top high schools in my country before I eventually got a new PC. This laptop also taught me the initial steps of exploring the Internet effectively and opened up a world of computer science knowledge.\nTo this day, even though I couldn\u0026rsquo;t keep the laptop because I donated it to my cousin, I still use the second monitor as it has become an integral part of my workflow. Since having two screens, my work productivity has significantly increased as I can multitask and access information more easily. In fact, if given the choice between an extremely high-end laptop with only one screen or a mediocre one with an additional screen, I would not hesitate to choose the latter.\nAfter I got into Amsterdam High School, here\u0026rsquo;s my mum\u0026rsquo;s present. # Boom, a big upgrade coming right there. Like, literally from a 50 USD budget laptop to a 700 USD high-end PC. Not to mention this is also the first time I have had the courage to ask my mom for a present, as I usually shy away from asking my parents to spend money on me. However, this time was different because for the first time in my life, my parents were ready to spend this much for my attendance at Amsterdam High School.\nHowever, as much of a tragedy it may seem, this is also the time when hardware components became so expensive due to the rise of Bitcoin and Cryptocurrency mining. So if I had the opportunity to go back, I would probably wait until November 2021 to buy a new one because with the same specifications, I could have spent only around $400 on this one. Still, I am so grateful for this because it was a big transition from my last laptop and it serves my needs way better than its capabilities.\nEverything got replaced except for the DELL second monitor mentioned above. I also took the old hard disk from my laptop to serve as the second hard drive for my computer. Truth be told, initially, I was very excited to set up my workstation. I often visited Facebook groups and watched videos about workspaces, cable management, decorative accessories, and more from various individuals. After countless times hiding under the table trying to make the cables look organized, I finally got the final setup shown above. However, I was too focused on aesthetics without actually realizing what was most important: productivity.\nAfter about three months of attending online Zoom classes where my bed and chair were so close together, I started developing bad habits. Whenever I felt tired or sleepy during class, it was too easy for me to just fall back onto that bed and have a sleepy \u0026ldquo;day\u0026rdquo;. It was good for my health but definitely not good for my studies, so I once again had to make a change.\nSeparate my \u0026ldquo;working me\u0026rdquo; apart from my \u0026ldquo;sleepy me\u0026rdquo; # The picture above that you see right here shows the setup I had been using for nearly two years. As you can see, there is a table serving as a divider between the chair and the bed. This setup prevents me from \u0026ldquo;sleeping\u0026rdquo; when I have to attend a Zoom class or pull an all-nighter. The table not only serves as a place to put additional things on but also separates my \u0026ldquo;work\u0026rdquo; mode from my \u0026ldquo;entertainment\u0026rdquo; mode.\nAdditionally, the second old monitor has now been turned into a vertical one since the space on the table has become narrower. However, the main reason for this change is because there is some content that I really love to view vertically, such as articles, coding documentation, or Phuong Ly\u0026rsquo;s videos -)). Not only can you view more content at once, but it also resembles a big smartphone, allowing you to view optimized social media sites just like when surfing on phones like Medium, Facebook, Reddit,\u0026hellip;\nHowever, it\u0026rsquo;s easy to have an Icarus feeling - never being satisfied with what you currently have. After using two monitors for a while, I have the urge to have three monitors because when I am learning Coursera MOOC courses or watching YouTube reference materials while coding together, the vertical ones really irritate me because they can\u0026rsquo;t scale fullscreen. But I still have to suppress this urge because I know that I don\u0026rsquo;t have enough money to purchase another one and my VGA doesn\u0026rsquo;t have three ports to export screens. Until then,\u0026hellip;\nI once again changed because now I have 3 monitors and a Hackintosh # This is probably the last update before I leave my home to attend university. As you may have anticipated, after 2 years, I finally got a \u0026ldquo;new\u0026rdquo; monitor from my mom\u0026rsquo;s office leftovers (I\u0026rsquo;m really grateful for the whole office there, you know).\nI also switched my operating system from Windows to MacOS, but instead of buying a brand new Mac, I \u0026ldquo;Hackintoshed\u0026rdquo; my computer. Additionally, I purchased a refurbished RX580 VGA from a Facebook marketplace because my old GT1030 had some compatibility issues with MacOS.\nAnd that\u0026rsquo;s the end of my workspace optimization journey. As you probably expected, having a more modern and optimized workspace can lead to increased productivity, right? Well, that\u0026rsquo;s not entirely true. I\u0026rsquo;ve come to realize that true productivity comes from within and is not solely dependent on the hardware or number of monitors you have.\nThe funny thing is that even though my laptop setup when I was in 9th grade had the lowest specifications, I felt second most productive compared to my different setups mentioned above. I guess it\u0026rsquo;s because there were fewer distractions with that setup; all I could do was study. On the other hand, with the high-end PC setup, it\u0026rsquo;s easy to get distracted and procrastinate with games and social media sites. (I\u0026rsquo;ve put in a lot of effort to counteract this distraction and someday I\u0026rsquo;ll write a blog sharing how I\u0026rsquo;ve fought against it). So here\u0026rsquo;s my current most productive setup, which you may not be ready for yet.\n","date":"4 November 2023","externalUrl":null,"permalink":"/posts/productivity-workspace/","section":"Posts","summary":"Today, I would like to introduce you to the journey of optimizing my workspace environment, or in simpler terms, my desk setup. In fact, I am not someone who strictly adheres to principles in life. However, I really enjoy working in a disciplined environment and always strive to optimize it for effective studying and working. To achieve the setup I have today, I had to tear down and rebuild this workstation more than 10 times. Finally, I am truly satisfied with it. It not only satisfies me aesthetically but also makes me eager to work as soon as I step into this room (well, not quite true because nowadays most of the time I feel most productive when working from a coffee shop…).\n","title":"Organizing My Digital Life - How My Workspace Evolved from Chaos to Hackintosh","type":"posts"},{"content":"","date":"20 October 2023","externalUrl":null,"permalink":"/tags/hackathon/","section":"Tags","summary":"","title":"hackathon","type":"tags"},{"content":"Today, I want to share my experience with you about the project I worked on during the Steam Hacks 2023 competition. Although my team did not win any major awards for this project, it was an incredible learning opportunity for me, particularly in terms of coding. Moreover, I also gained valuable insights into the collaborative process through working with my amazing new friend from Son La - Minh Chau.\nBackground information about STEAM Hacks # STEAM Hacks is a national hackathon organized by STEAMS for Vietnam in collaboration with Ha Noi University of Science and Technology. It offers two tracks, Hipster and Hacker, catering to UX/UI designers and developers respectively.\nAlthough I initially had confidence in joining the Hacker Track, I ultimately decided to participate in the Hipster Track. This choice was driven by my desire to have better overall project management skills and to further enhance my UX/UI abilities. However, it turned out that I still had to write code for the team despite being in the Hipster Track. (you would know the reason later on this article)\nThe hackathon consists of three rounds. The first round (Breaker Challenge) is an individual challenge with separate tracks: Hipster and Hacker. Each track would receive support by various workshops, covering the knowledge required to participate to the competition. At the end of the first round, participants from each track are required to submit an assignment that showcases their newly acquired knowledge. For Hipsters, this involves designing an e-commerce landing page with a high-fidelity wireframe created on Figma, along with user personas and user journey details related to their product. Hackers must submit a functional website developed using Flask framework and incorporating some form of \u0026ldquo;AI\u0026rdquo; feature.\n150 participants from both tracks are then chosen for Round 2: Innovation Spirit, where they form teams of 2 Hipsters and 2 Hackers and choose from four tracks: Sustainability, Education, Mental Health, or Community. Only 15 teams advance to the final round, The Hacking Day, with 7 team awards and 2 individual awards. It seems like I came close to being recognized as one of the best Hipsters since I was invited for an interview among the Top 5 Best Hipsters, but I couldn\u0026rsquo;t make it in the end though 😢 (interestingly, the recipient of the Best Hipster Awards was none other than Minh Chau, a fellow member of my team)\nMy initial idea for both first and the next two round # After placing as a Top 10 Finalist in last year\u0026rsquo;s Samsung competition, I couldn\u0026rsquo;t help but feel a sense of dissatisfaction with my product. It lacked certain features I wanted to include, and there were numerous unresolved issues with the app\u0026rsquo;s development. This drove me to participate in this competition, with a focus on creating another application to finish an overall optimized system for reducing e-waste. In the initial round, where I had to design a landing page for an e-commerce site, I immediately decided to develop a marketplace for trading refurbished devices. This solidified my determination to craft a comprehensive product idea for the subsequent rounds, even though I wasn\u0026rsquo;t certain of advancing beyond round 1.\nRecognizing the importance of further developing this idea, I dedicated significant time to meticulously designing wireframes for the project. However, due to personal circumstances, I found myself dangerously close to missing the round 1 deadline. Thankfully, only by submitting the final version just 30 minutes before the cutoff did I manage to make it on time (Pheww). You can access my completed assignment here. This also qualified me for entry into the next round. Additionally, I had the honor of being recognized as one of the top five Best Hipster participants in the competition because of my performance on this round.\nRound 2 - The bond and conflict arise # Upon receiving the email notifying me about the results and next steps, I also discovered that my old friend - Tu Linh - had also participated in the competition and qualified for the second round. The only difference was that he had chosen to join the Hacker track. After a brief discussion, I decided to join him along with two others: his friend Huy, and a randomly recruited girl named Minh Chau. While the three of us were from Hanoi and Minh Chau hailed from Son La, it was her who took the initiative throughout our collaboration (and also contributed the most). That was also the reason why she became a leader for my team. Well, she is really good at project management and business analytics, but underperforming in software development. That also the time I became the Software Development Lead\nThe initial meeting went smoothly as we all introduced ourselves and comfortably shared our ideas from the first round for brainstorming purposes. While I don\u0026rsquo;t want to come across as boastful, it became apparent to me that my assignment was the only one with the potential to win major awards. Huy and Linh from the Hacker Track supported my viewpoint. They agreed that my already beautiful interface and adherence to guidelines would significantly reduce the time spent on front-end development. However, Minh Chau had some reservations. She did like my idea, but she stated that it just so \u0026ldquo;ordinary\u0026rdquo; and \u0026ldquo;unsafe\u0026rdquo;. After several more meetings filled with debates and discussions, we eventually reached a consensus on what we should create for rounds 2 and 3: a counterfeit product verification system.\nYou might be wondering, is that all? Why were you so easily compromised? Well, I did attempt to negotiate with her. However, as mentioned earlier, choosing the idea of creating a marketplace for refurbished devices would result in my team being disqualified from the STEAM Hacks competition if they discovered any correlation or signs of idea copying from my Samsung product last year. On another note, my primary motivation for participating in this competition was not solely to complete my Samsung product but rather to acquire and expand my knowledge and skills.\nBuilding the entire application myself # I don\u0026rsquo;t want to point fingers or be too hard on myself, but it\u0026rsquo;s clear that none of the Hacker members on our team were pulling their weight. It\u0026rsquo;s not entirely their fault though; they excel in competitive coding, but they weren\u0026rsquo;t adequately prepared for web development or a hackathon like this. So, I found myself having to join them in tackling the code development.\nFurthermore, our team was currently facing a conflicting idea with our leader regarding the counterfeit product verification system. Unlike other student projects that typically involve education or mental health, this idea feels challenging for us to create something meaningful. Initially, our leader proposed using AI to detect fake patterns or incorrect labels, but our team lacks the competence in developing AI-related features. Additionally, implementing such a system seemed impossible and unproductive due to the vast variety of products and their unique fake versions. Moreover, there are also concerns about implementation costs and potential returns.\nWell, turns out, every cloud has a silver lining. Firstly, despite my limited technical knowledge at that time, I embraced the opportunity to learn backend development on my own, driven by a strong determination and purpose. This made me feel alive and empowered. Specifically, I was fortunate to have access to the FlutterFlow Education program, which enabled me to quickly learn and utilize their user-friendly software for creating meaningful applications. This greatly reduced our front-end development time and expanded my knowledge of this powerful tool. Furthermore, I acquired proficiency in Flask to develop an API backend application that automatically serves ML models trained through the Teachable Machine platform. This saved me the tedious task of manually researching and collecting data for training models.\nHowever, it is easier planning than implementing.\nExploring the Implementation Process # During the initial stages, I explored various potential implementations, each presenting its own set of hurdles. I want to make an Android application which have the Computer Vision feature on it. Initially, I came up with the idea of hosting a .Tflite model offline using Android Studio, which seemed promising, but the dependency management and debugging complexities proved daunting. Despite investing countless hours to resolve library errors, achieving a successful attempt remained elusive. Similarly, attempting to host a Transformer model on HuggingFace appeared overly complex, demanding a deep understanding of Transformer and Tensorflow intricacies.\nHowever, the breakthrough emerged with the decision to develop a Flask backend for a REST API server, specifically tailored to handle Teachable Machine exported models. This decision, inspired by a helpful tutorial I stumbled upon, paved the way for a more feasible and practical approach.\nLink of the tutorial: https://sogalanbat.medium.com/custom-api-for-keras-model-using-cloud-run-9d367a2ea5e8\nOvercoming Challenges and Constraints # Finding a way to accomplish my goal was a challenge, but I persevered. However, when the time came to put my plan into action, I realized it was much more difficult than anticipated. At the outset, my lack of prior knowledge in Machine Learning Model and Python Flask REST API posed a steep learning curve. Existing tutorials primarily focused on deploying entire Flask web apps or Tensorflow models, which didn\u0026rsquo;t directly address the Teachable Machine\u0026rsquo;s unique requirements.\nFurthermore, wrestling with outdated Python packages presented a considerable setback. Efforts to convert image URLs directly to tensor files proved unreliable, often yielding inaccurate results due to the limitations in handling diverse image file types. However, persistence and patience ultimately led me to discover a workaround. I learned how to download image URLs directly and convert them to the compatible Keras format, effectively resolving the earlier challenges.\nThe deployment phase, while initially daunting, became more manageable with the aid of a comprehensible Dockerfile tutorial. Thanks to this newfound understanding, I successfully deployed the application on Google Cloud Platform (GCP).\nDespite the successful implementation, certain limitations persist. The machine learning model\u0026rsquo;s inability to recognize products with 100% accuracy remains a challenge. Additionally, scaling the model to encompass a wider range of products proves challenging due to constraints in data resources. Although there were still flaws and room for improvement, we managed to qualify for the next round - the Hacking Day\u0026hellip;\nThe application cannot detect these physical patterns on real and fake products.\nAlthough it achieved a 70% success rate after conducting 400 tests, relying solely on this mechanism may not be deemed reliable.\nOverview of our app before the final round\nFinal revision and New trajectory # Entering the final round was unexpected for all of us. Still, we have to figure something out to improve the process of verifying counterfeit products.\nTo address these limitations and optimize the user experience, a novel approach has been conceived. Instead of relying solely on the machine learning model to identify differences between fake and authentic products, the application will facilitate user interaction with informative content. Users will receive guidance on how to physically examine the product, potentially by scanning barcodes or utilizing Optical Character Recognition (OCR) to decipher product numbers. The extracted data will then be passed through ChatGPT (LangChain) to source relevant internet pages based on credibility and popularity. A summarized report will then be relayed back to the user, empowering them to make informed decisions.\nThe day before the final, we had a meeting scheduled with our counselor and advisor. It was embarrassing because our team had barely finished the final version of our product. However, this gave us an opportunity to ask our advisor important questions regarding the challenges we were currently facing. As I delved deeper into NLP and specifically LangChain, I discovered that it was poorly developed and relied heavily on tokenization. After a lengthy discussion, we also learned some valuable techniques for optimizing it. And in the same day, we were able to complete our underperforming product, making it ready for the presentation scheduled for the following day.\nOn the day of our presentation, nothing out of the ordinary occurred. However, one memorable moment stands out in my mind. Our team made the decision to demonstrate the live usage of our application. Unfortunately, it didn\u0026rsquo;t go as planned and resulted in uproarious laughter from the audience. Despite this setback, we were able to complete the presentation without further complications (although our question and answer session left much to be desired). Although our team did not receive any awards in the competition, we still managed to capture a photo with smiles all around.\nHere\u0026rsquo;s a video demonstration of the app.\n","date":"20 October 2023","externalUrl":null,"permalink":"/posts/hackathon-steam-hacks-2023/","section":"Posts","summary":"Today, I want to share my experience with you about the project I worked on during the Steam Hacks 2023 competition. Although my team did not win any major awards for this project, it was an incredible learning opportunity for me, particularly in terms of coding. Moreover, I also gained valuable insights into the collaborative process through working with my amazing new friend from Son La - Minh Chau.\n","title":"My STEAM Hacks 2023 experience","type":"posts"},{"content":"","date":"20 October 2023","externalUrl":"https://github.com/phuchoang2603/techburst-counterfeit","permalink":"/projects/steam-hacks-2023-techburst-counterfeit/","section":"Projects","summary":"This project aims to provide users with a reliable solution to identify counterfeit products. Originally designed as a C2C Ecommerce Platform, the project has evolved to focus solely on counterfeit detection. The system offers two methods for users to verify product authenticity: utilizing a Teachable Machine model and employing a barcode scanner.","title":"TechBurst – Revolutionary AI Solution for Identifying Counterfeit Products","type":"projects"},{"content":"It all started with a computer my mom saved from the trash heap at her office. It was a standard HP Prodesk 600 G4—nothing special, and since I already had a PC and a laptop, I had no idea what to do with it. Just installing Windows on it felt like a waste. I knew it could be something more than just another desktop collecting dust.\nThat simple idea kicked off a journey that completely changed what I thought I was passionate about and what I wanted to do with my life.\nMy First, Clumsy Steps # My first goal was to create my own private server to back up all my photos. I stumbled upon something called Xpenology, which is a clever way to run the software from those fancy, expensive Synology servers on any old computer. I spent a whole night fighting with it. It was a huge pain. The whole thing relied on a specific USB stick that had to be plugged in all the time to trick the software into working. It felt messy and unreliable.\nhttps://voz.vn/t/huong-dan-build-case-nas-xpenlogy-dsm-7-2-full-license-synology-surveillance-9-1-2-tan-dung-pc-cu.845346/\nFrustrated, I kept searching and found Proxmox. This was the \u0026ldquo;aha!\u0026rdquo; moment. Proxmox is an operating system built specifically to run other computers inside of it. Think of it like a digital nesting doll. It was stable, powerful, and let me create a virtual \u0026ldquo;computer\u0026rdquo; just for my photo backups. I could finally experiment and mess around without the fear of crashing everything. The beast was finally tamed.\nFinding a Real Purpose (With a Little Help from a GPU) # The server was running, but I knew it could do more. Things got really interesting when I found an old, barely-working GT 1030 graphics card. It was the kind of thing most people would throw out, but I stuck it in the PC, and suddenly, a whole new world of projects opened up.\nFirst, I built my own personal Netflix. I used a free program called Jellyfin to organize all my movies and TV shows into a slick interface. To get the media, I set up a few helper tools that people call the \u0026ldquo;arr stack\u0026rdquo; (Sonarr for TV, Radarr for movies) that automatically find and download stuff for me. That little graphics card did all the heavy lifting, converting video files on the fly so I could watch anything on my TV, laptop, or phone without any stuttering. If you want more detailed on how to properly set this up, here\u0026rsquo;s the link https://trash-guides.info/.\nNext, I got into AI before ChatGPT was cool. I found a program called Stable Diffusion that can create images from just text descriptions. I had a pretty juvenile idea for a Telegram bot: you send it a picture, which then trigger a web-hook to a script I wrote to identify people clothes and send it to the Stable Diffusion server, which then \u0026hellip; well, let\u0026rsquo;s just say \u0026ldquo;creatively redraw\u0026rdquo; the clothing. It was a ridiculous project born from a 16-year-old\u0026rsquo;s hentai brain, but it taught me a ton about programming and how to connect different services together. Here\u0026rsquo;s the repository if anyone interested https://github.com/phuchoang2603/telegram-sdwebui.\nBut the most meaningful thing I built was for my family. I connected our existing security cameras to a program called Frigate. Using the graphics card, Frigate is smart enough to tell the difference between a person walking up to our door and just a tree swaying in the wind. I then hooked Frigate into Home Assistant, which is like a central brain for smart home gadgets. Now, if someone is walking around our house late at night, my entire family gets an alert on their phones with a picture of what’s happening. It’s given us some real peace of mind, all powered by that old office PC.\nThe Obsession and the Path Forward # I was officially hooked. I found online communities like the r/selfhosted and r/homelab subreddits and discovered I wasn\u0026rsquo;t alone. There are tons of people out there building amazing things in their own homes.\nAt first, all my projects only worked inside my house network. But what if I wanted to watch a movie while I was out, or share my photo server with a friend? I learned how to use something called a Cloudflare Tunnel. It sounds complicated, but it\u0026rsquo;s a super secure way to let my projects be seen by the wider internet without opening up my home network to hackers.\nLooking back, it’s wild. This whole journey started with a free computer that was about to be scrapped. In the process of tinkering, I accidentally taught myself about networking, Linux servers, and coding. I wasn\u0026rsquo;t just building a server; I was discovering something I truly loved to do. It made me realize I didn\u0026rsquo;t want to be a UX/UI designer anymore. I wanted to be the person who builds the stuff that works behind the scenes.\nThat old HP Prodesk didn\u0026rsquo;t just find a new purpose; it gave me one, too.\n","date":"27 August 2023","externalUrl":null,"permalink":"/posts/self-hosted-initial-story/","section":"Posts","summary":"It all started with a computer my mom saved from the trash heap at her office. It was a standard HP Prodesk 600 G4—nothing special, and since I already had a PC and a laptop, I had no idea what to do with it. Just installing Windows on it felt like a waste. I knew it could be something more than just another desktop collecting dust.\n","title":"My First Server Was Office Trash: A Self-Hosting Story","type":"posts"},{"content":"This is my product created while participating in Samsung Solve for Tomorrow 2022, a STEM contest organized by Samsung Electronic Company that encourages problem-solving skills and showcases the positive impact of young minds on society, with over 76,000 participants nationwide\nDemo # User experience case study # Website # Built during the first round of STEAM Hacks 2023, Reware Website is a C2C marketplace inspired by platforms like eBay and Shopee. It enables users to buy and sell refurbished electronics and features self-serve kiosks for convenient device trade-ins. Reware promotes sustainable tech use by giving old devices a second life while offering users cash incentives.\n","date":"9 November 2022","externalUrl":null,"permalink":"/projects/sft2022-reware-app/","section":"Projects","summary":"This is my product created while participating in Samsung Solve for Tomorrow 2022, a STEM contest organized by Samsung Electronic Company that encourages problem-solving skills and showcases the positive impact of young minds on society, with over 76,000 participants nationwide\n","title":"Reware App – Ewaste Management Solutions","type":"projects"},{"content":"This is my diary about the trip to Sapa Hope Center with 10 colleagues from Summit Education - a renowned English language teaching center in Hanoi. Therefore, this trip is aimed at fostering cultural exchange and teaching English and basic skills to students from Grade 1 to Grade 2 in Sapa, Lao Cai.\nDay 1 # After many days of anticipation for new tasks and challenges – my first time as a cameraman – on the charity trip to Sapa with Summit, today finally arrived. I woke up quite early, having only slept for 3 hours the night before. I should have slept more, but I woke up at 3:40 am. After brushing my teeth, I fumbled on my computer to download videos on photography tutorials from YouTube to watch and learn on the way (I was assigned to take photos even though I\u0026rsquo;ve never held a camera before -))). I downloaded the videos and finished preparing my gear by 4:40 am, then called my dad to drive me.\nUpon arriving at the gathering point, I realized I was early af -)) However, that period was also when I started to play around and learn more about the camera. It was quite fascinating, to be honest, because like everything else, it has its own rules and standards that we can learn in a simple and understandable way. Having said that, after getting on the bus and starting to watch the videos I downloaded earlier, I only got a basic grasp of concepts like lens mm, aperture, shutter speed, ISO, \u0026hellip; And I haven\u0026rsquo;t learned anything about framing angles and positioning yet.\nAt Sapa Hope Center, the most surprising thing for me was the extremely warm welcome from the kids. Although many of them couldn\u0026rsquo;t speak Vietnamese (Sapa is a land of convergence for 5 different ethnic groups), they were enthusiastic and innocent. As soon as I arrived, the kids gave each of us a flower and a welcoming card, which they had drawn themselves. The drawings looked simple, but I believe they came from the sincere and innocent hearts of these little ones.\nDuring the self-introduction, I was quite surprised when the owner of the center turned out to be a young man (seemingly stepping out of \u0026ldquo;Silent Sapa\u0026rdquo;) who had studied abroad in Singapore. After visiting this place, he fell in love with it, despite numerous opportunities in the city. Even his relatives thought he was \u0026ldquo;charmed\u0026rdquo; (there is a local belief in enchantment being quite common here). He shared many stories about Sapa – the specially nutritious herbs, the underprivileged children lacking sufficient food, the uphill journey from home to school taking 2-3 hours each day, and more. Peter Thương, along with the kids at the center, made me feel moved every time I heard those genuine and simple stories.\nAfter having lunch and a break, we spent the entire afternoon getting to know each child individually to better understand their personalities. This helped us plan and manage the teaching program more effectively for the upcoming days. Despite having only known each other for a few hours, we engaged in drawing exercises, played games, and sang together, creating a strong bond. The room that day was filled with laughter.\nThe entire group introducing themselves and playing games\nThe older members teaching the children how to draw\nSharing and chatting together\nPlaying the \u0026ldquo;musical chairs\u0026rdquo; game together\nAfter finishing the activities, the sky had darkened, and we were quite tired from the day of travel. We decided to return to the hotel to rest.\nTaken at Praha Hotel, 86 Violet Street, Sapa\nDay 2 # On that morning, instead of going to the large activity hall like yesterday, our group had to go to a dining place to set up a projector to assist in teaching the children. The reason for this was that our goal for the day was to teach the kids English through themed movies and songs, requiring a dark room to make it easier for the children to see.\nDuring the morning teaching session, we played songs and shared stories about the movie Frozen, then guided the children to fill in blanks or answer questions about the movie content. However, it seemed that some of the children had already seen the movie before, and for some, it was a challenging task. Despite our efforts to maintain their interest in the first 1-2 hours, we couldn\u0026rsquo;t make that teaching session as effective as we hoped.\nAfter lunch, Chau Anh (from our group) decided to show the kids a famous Disney movie about three squirrels to regain their interest. However, after 15 minutes of screening, the kids seemed bored with the movie and began to leave the room. Faced with this situation, our group decided to stop showing movies and return to the large activity hall, directly interacting and teaching the kids how to create and play games.\nIndeed, the children here were too accustomed to watching movies whenever there was a foreign charity group visiting, so they had become disinterested in watching another movie. Instead, they enjoyed running and playing with everyone – something they rarely get to do when charity groups visit. Therefore, when given the opportunity to play freely that day, the adorable kids at Sapa Hope Center were very happy and cooperated completely with the older members, creating laughter for both sides.\nThe older members playing games designed by the children\nAfter the play session, we presented books, notebooks, and clothes to the kids Upon returning to the hotel, we held a meeting with the staff to address the issue of the \u0026ldquo;lesson plan burnout\u0026rdquo; that occurred that day. However, what I remember most from that evening was the conversation between me, Ms. Mai, and Chị Minh Hà\u0026rsquo;s mother. Ms. Mai shared a lot about the difficulties and experiences in understanding the psychology of teaching preschool and elementary school children. (There was one saying from her that I found very insightful: \u0026ldquo;It takes losing nine toes to finally realize we need to protect the remaining one.\u0026rdquo;) Chị Minh Hà\u0026rsquo;s mother also shared a lot about life in Russia.\nDay 3 # On that morning, we continued our teaching sessions for the children at Sapa Hope Center. The goal for our group on this day was to educate the children about personal hygiene skills through lively videos and practical exercises.\nHowever, unlike the previous day, this time, we decided to use the large activity hall (rather than the dining area, which was too dark and dirty by evening) for the lecture and for the convenience of teaching in a more dynamic manner (as discussed and planned at the hotel after reflecting on the previous teaching session). Indeed, moving to a spacious and clean room, as shown in the picture, significantly improved the comfort for both the teachers and the children.\nAdditionally, we must acknowledge the enthusiasm of the two \u0026ldquo;teachers,\u0026rdquo; Nhi and Linh, who brought interesting lessons on personal hygiene to the children. The two of them, along with the entire group, collaborated to guide the children in dancing to songs, creating artwork, and participating in related games. Later, the children practiced by taking fun quizzes, such as coloring objects corresponding to their daily hygiene items.\nWhile I wasn\u0026rsquo;t the main photographer for the morning session, I felt delighted to participate in a different role as a support person, helping the children complete their tasks in a joyful and easy manner.\nAfter completing the teaching session at the center, in the afternoon, our group planned to visit each child\u0026rsquo;s home to better understand their family situations and daily lives. At each home, we spent some time listening to the children\u0026rsquo;s stories and the family\u0026rsquo;s struggles. It was an opportunity to gain insight into the hardships they face and learn about the various professions within each household.\nReflection # It can be said that, although this trip is not too short or too long, it is sufficient to allow me to experience, try new things, and learn. There were many firsts for me on this journey: the first time participating in a charity trip without my parents, the first time having to plan and adapt to everything on my own, the first time being the one behind the camera capturing moments and actions throughout the trip, and more. In addition to the experiential lessons from these firsts, I also met and learned a lot from the children, older members, and staff. Everyone helped me engage and listen to many unique and diverse stories. It can be said that this trip will probably leave a deep memory during my high school years.\n","date":"30 June 2022","externalUrl":null,"permalink":"/posts/volunteer-sapa-hope-center/","section":"Posts","summary":"This is my diary about the trip to Sapa Hope Center with 10 colleagues from Summit Education - a renowned English language teaching center in Hanoi. Therefore, this trip is aimed at fostering cultural exchange and teaching English and basic skills to students from Grade 1 to Grade 2 in Sapa, Lao Cai.\n","title":"Lesson Learned in Three-day Trip to Sapa Hope Center","type":"posts"},{"content":"This product emerged from my participation in the BASF Innovation Challenge, a national competition that empowers students to devise sustainability solutions, where we secured the 3rd Runner Up prize. While our aim was not the highest award, this competition established a robust foundation for me and our team to persist in researching and developing solutions to address the serious issue of e-waste\nPitching # Applications User Interface # ","date":"9 June 2022","externalUrl":null,"permalink":"/projects/basf-green-express/","section":"Projects","summary":"This product emerged from my participation in the BASF Innovation Challenge, a national competition that empowers students to devise sustainability solutions, where we secured the 3rd Runner Up prize. While our aim was not the highest award, this competition established a robust foundation for me and our team to persist in researching and developing solutions to address the serious issue of e-waste\n","title":"The Green Express – Initial Business Concept of Fostering Ewaste Reduction","type":"projects"},{"content":"","externalUrl":null,"permalink":"/authors/","section":"Authors","summary":"","title":"Authors","type":"authors"}]