diff --git a/build.sh b/build.sh index c235b23..c83aaf4 100755 --- a/build.sh +++ b/build.sh @@ -1,2 +1,2 @@ -docker build . -t git.howdoicomputer.lol/howdoicomputer/blog:5 -docker push git.howdoicomputer.lol/howdoicomputer/blog:5 +docker build . -t git.howdoicomputer.lol/howdoicomputer/blog:7 +docker push git.howdoicomputer.lol/howdoicomputer/blog:7 diff --git a/ci/main.go b/ci/main.go index b2b8a8a..23877c3 100644 --- a/ci/main.go +++ b/ci/main.go @@ -11,23 +11,20 @@ import ( func main() { ctx := context.Background() - // initialize Dagger client - client, err := dagger.Connect(ctx, dagger.WithLogOutput(os.Stdout)) + client, err := dagger.Connect(ctx, dagger.WithLogOutput(os.Stdout), dagger.WithWorkdir("..")) if err != nil { panic(err) } defer client.Close() - // use a golang:1.19 container - // get version - // execute - golang := client.Container().From("golang:1.19").WithExec([]string{"go", "version"}) + buildDir := client.Container(). + From("klakegg/hugo:0.111.3-ext-onbuild"). + Directory("/target") - version, err := golang.Stdout(ctx) - if err != nil { - panic(err) - } + ref, err := client.Container(). + From("nginx"). + WithDirectory("/usr/share/nginx/html", buildDir). + Build() - // print output - fmt.Println("Hello from Dagger and " + version) + fmt.Println(ref) } diff --git a/content/posts/dagger-pe.md b/content/posts/dagger-pe.md new file mode 100644 index 0000000..ce86495 --- /dev/null +++ b/content/posts/dagger-pe.md @@ -0,0 +1,51 @@ +--- +title: "Dagger: Platform Engineer's Dream" +date: 2023-11-05T19:53:24-08:00 +draft: true +--- + +# but what is dagger + +Dagger is a multi-language toolkit that allows you to create CI/CD pipelines that are backed by Docker. Imagine if `make` was an API where you could write targets in Go, TypeScript, Python, etc and then each target was ran inside a Docker container and the results of each container was cached. + +For example, let's say we have a web API written in Rust. For this hypothetical project, we want to test, build, upload, and deploy. Test would be running `cargo test`, build would be either creating a container image or an executable, upload would be uploading a container image to a registry, and deploy would be running `helm upgrade` to upgrade an existing Kubernetes `Deployment`. + +With Dagger, each of those steps could be written using Rust, with each step executing inside a container image built for that step, all running on a developers laptop. When that pipeline needs to be slotted into Jenkins or any other CI, all the CI system needs to be able to do is run Dagger. + +But we aren't just limited to a make target analogue. Dagger pipelines are written using a REAL programming language - not a DSL! That means anything is possible! Do you need to fetch a secret from a secret store? Do you need to support a custom versioning process using git tags? Do you need to communicate with internal services when deployments occur? You can do all of these things using a language that you're comfortable with. Gone are the days of hunting down long abandoned Jenkins plugins written in Groovy and riddled with security vulnerabilities just to add simple functionality to your pipelines. Write it in a language you love instead. <3 + +# internal dsl library + +At a certain scale, it's not uncommon for a company to have a blessed path for developers to onboard their applications and processes into a CI runner like Jenkins, TeamCity, or GitHub Actions. This typically coalesces as a singular library that a platform engineering or release engineering team builds and then provides to the greater engineering org as a sort of an SDK for CI/CD processes. If your company is running Jenkins, for example, you might have a `jenkins-utilty-lib` that gets imported into DSL files that are stored in application repositories and that utility lib would have functions for enforcing company standards or abstracting complex release processes. + +Dagger can replace those DSL libraries with code that the platform engineering team favors rather than forcing them to write Groovy. But... what if they choose isn't favored by everyone else? + +# any language, comrade + +Okay, so internal DSL libraries are common and Dagger allows you to write them in any language. But what if your platform engineering team really wants to code in Go but your front-end engineers really want to code in TypeScript? What if the PE team writes some Go functions in Dagger that uploads static assets to a CDN? Does the FE team need to write Go to use those functions? + +Well, no. + +Dagger has the ability to expose pipeline steps **across languages**. As long as the output of a function is serializable, it can be consumed by TypeScript from Go (or any other pairing). In the above example, the platform engineering team *can* write build pipelines that are compatible with front-end's pipelines even if they aren't in the same language. + +# but what about CLI tools + +So, for example, let's say we have a CLI tool written in Ruby and it requires a specific set of libraries installed and a sane Ruby environment. This Ruby CLI could be some internal tool used by your company to automate some process. However, it's kind of unwieldy to have every engineer install Ruby on their machine alongside this fantasy tool's dependencies. + +With Dagger modules, you could easily provide a way for end users to consume your API by not having them install dependencies. + +Using a [Python dagger module](https://daggerverse.dev/mod/github.com/helderco/daggerverse/python@ba7984f47857794ff5aace82947aaf6b433595be) as an example: + +``` +dagger mod install github.com/helderco/daggerverse/python@ba7984f47857794ff5aace82947aaf6b433595be +export DAGGER_MODULE=github.com/helderco/daggerverse/python +dagger up http-server -n --src . +``` + +The above commands will install the Python Dagger module and then use it to spin up a Python container and run `http-server` to serve files. + +# caching, k8s, and other good stuff + +Because Dagger uses Docker BuildKit under the hood it also has support for caching operations and artifacts from pipeline steps. Additionally, you can run the Dagger Engine on Kubernetes and have it run your CI/CD so that the build caches are shared across multiple CI runs. This means that there is a TON of opportunity for time savings across multiple runs. + +Seriously, check it out. It's neat. diff --git a/content/posts/homelab-1.md b/content/posts/homelab-1.md index f37fa1a..216e4e5 100644 --- a/content/posts/homelab-1.md +++ b/content/posts/homelab-1.md @@ -14,8 +14,6 @@ This blog. The one that you are currently reading right now. Is running on a met ### yeah, but why -I like solving problems and these were mine: - * I have a plethora of software projects that I've written that need a home. Discord bots, daemons, websites, etc. * I needed a place to host a blog. Hi! * I want to replace the myriad online software services that I'm subscribed to. Spotify, Netflix, etc. @@ -54,7 +52,7 @@ Here are some features of my homelab: A secret that I've withheld from you so far is that Megamind isn't my first homelab. A previous homelab that I built was a k3s cluster running on top of Raspbery Pis. It was quaint but got decommissioned when I moved apartments. I never set it up again and it currently lies in pieces in a drawer somewhere. -So why Nomad? Mostly because, at the time, I was leading an infrastructure initiative at my day job to use Nomad to orchestrate applications that are deployed to heterogenous IoT devices spread across distinct geographic regions. I wanted to dog food my own proposal. Also, Nomad is *a lot* simpler to use than Kubernetes. Nomad starts as *just* an orchestrator but gives you the option to introduce complexity later on through Vault and Consul. Every concept is almost immediately grokable and it has just been a pleasure to work with. +So why Nomad? Mostly because, at the time, I was leading an infrastructure initiative at my day job to use Nomad to orchestrate applications that are deployed to heterogenous IoT devices spread across distinct geographic regions. Nomad was a better fit than k8s for that specific use case because it was lighter-weight, easier to use, had less overhead (which was important because embedded devices don't have a lot of resources), and could run non-containerized applications that needed deep access to hardware. And so I wanted to dog food my own proposal. Also, Nomad is *a lot* simpler to use than Kubernetes. Nomad starts as *just* an orchestrator but gives you the option to introduce complexity later on through Vault and Consul. Every concept is almost immediately grokable and it has just been a pleasure to work with. ### what's runnin @@ -162,6 +160,6 @@ The future is bright! It has been incredibly fluid to roll out new services to t For CI/CD, I'll most likely go with [Drone](https://www.drone.io/). Really, I just want a UI for Dagger pipelines. For an auth model, eh, I am not sure. LDAP? There is also the possiblity of using Oauth2 and whitelisting accounts. -I also want to do a thorough lockdown of Gitea so that I can expose my code to the world. +For now though, my homelab is now an integral part of my life. For consumption, I use it to store a digital library of books, music, and movies. For productivity, the homelab is a cornerstone of developing applications that I want to expose to the public internet. It's... neat. -Till next time. +[Part 2]({{< ref "/posts/homelab-2" >}} "Part 2") diff --git a/content/posts/homelab-2.md b/content/posts/homelab-2.md index a322498..b5e9f0d 100644 --- a/content/posts/homelab-2.md +++ b/content/posts/homelab-2.md @@ -1,7 +1,7 @@ --- -title: "Homelab 2: Monitoring" +title: "Homelab 2: Monitoring Boogaloo" date: 2023-09-01T21:15:40-07:00 -draft: true +draft: false summary: How metrics and logs work for my homelab tags: - Prometheus @@ -13,9 +13,12 @@ tags: Every good production environment needs robust monitoring in-place before it could be considered production and I feel like my homelab shouldn't be an exception. -With that in mind, I created a monitoring stack that I've used professionally at companies that I've worked at and I'm pretty happy with it. +With that in mind, I created a monitoring stack that I've used professionally and I'm pretty happy with it. -## stackem +![dash](/node_exp_dashboard.png) +Grafana dashboard for host resource metrics + +## stackem' The different components of the system are thus: @@ -27,17 +30,72 @@ The different components of the system are thus: Here is what I want to collect metrics for, from most critical to least: 1. The base host resources. This includes available memory, CPU, disk space, network traffic, etc. Also includes the ZFS pool. -2. Nomad. -4. The services that run on the host via Nomad. +2. Nomad itself. +4. The services that are orchestrated by Nomad. -In essence, monitoring starts for the platform used to deliver my applications and then moves upward to the end services I expose to myself and my friends. +## tldr prometheus + +I won't go too deeply into how Prometheus works; I wouldn't be able to do better than the [official documentation](https://prometheus.io/docs/introduction/overview/). That being said, it's worthwhile to do a quick summary: + +* Applications are responsible for presenting their own metrics over HTTP via a dedicated endpoint +* Prometheus operates on a pull based model. As in, Prometheus will reach out and HTTP GET that endpoint for each application in order to "scrape" those metrics and store them +* Prometheus has a massive amount of support for different types of service discovery for automatically registering scrape targets +* Prometheus is a time series database (TSDB) - it is built to store linear, time oriented data that has dimensionality expressed through key-value pairs ## base system monitoring -Starting from the base, the Prometheus project supports the collection of metrics for Linux based hosts through their [node_exporter](https://github.com/prometheus/node_exporter) project. It's a Golang binary that exposes an absolute treasure trove of data including ZFS stats! It covers my first two monitoring needs. +For monitoring the base system, the Prometheus project supports the collection of metrics for Linux based hosts through their [node_exporter](https://github.com/prometheus/node_exporter) project. It's a Golang binary that exposes an absolute treasure trove of data... including ZFS stats! -While I could run the `node_exporter` via systemd, I instead opted to use the `exec` driver for Nomad. There is a Nix package for installing the exporter so the job definition just relies on executing the binary itself. Doing it this way means I get visibility into the exporter process and logs via the Nomad dashboard. +The exporter project recommends running it outside of a container as it needs a deep access to the host system and the isolation of containers are counterintiutive to that level of access. -## everything else +While I could run the `node_exporter` via systemd, I instead opted to use the `raw_exec` driver for Nomad. There is a Nix package for installing the exporter so the job definition just relies on executing the binary itself. Doing it this way means I get visibility into the exporter process and logs via the Nomad dashboard. -Nomad and Consul both expose their own metrics and so it's easy enough to add a Prometheus stat +``` +task "node-exporter" { + driver = "raw_exec" + + config { + command = "/run/current-system/sw/bin/node_exporter" + } +} +``` + +There is some security implications for enabling the `raw_exec` driver for Nomad as it will run every process under the same user that the Nomad client is running as - which is usually root. However, it was the path of least resistance and it's a TODO to later use the `exec` driver - which uses cgroups and chroot to isolate the spawned process. This, again, is counterintuitive to the node exporter being able to collect data and would require me to make sure that the task allocation has the correct access to the right resources in order to facilitate complete data collection. + +## service discovery brings all the metrics to the yard + +Prometheus has support for a plethora of service discovery methods that can be used to help it locate endpoints for scraping. The integrations range from using simple files to integrating with service meshes. For my homelab, I opted to use the [consul](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#consul_sd_config) service discovery integration because, well, that's what I'm running. + +Whenever a service is deployed to Nomad, it will register an IP address and port to Consul. Prometheus will then talk to Consul to get that address to construct a metrics endpoint and then start scraping metrics an a specified interval. + +The configuration for Prometheus looks like this: + +``` +scrape_configs: + - job_name: 'service_metrics' + consul_sd_configs: + - server: '{{ env "NOMAD_IP_prometheus" }}:8500' + relabel_configs: + - source_labels: [__meta_consul_service] + target_label: service +``` + +Since everything is on the same box, there is no authentication method for Consul and so the only necessary config is the server endpoint. The `$NOMAD_IP_prometheus` is the IP address for the Prometheus task - which is also the IP address for the homelab server itself because I make liberal use of the `bridge` networking type. As a quick note, bridge here means that all tasks share a networking namespace but ingress happens over a port bound to localhost for the Nomad client. + +The relabel config exists to take the Consul service source label - in this case, the Consul service label is the name of the service that is running on Nomad - and create a target label of `service` so that every metric is properly tagged. This means that if I deploy a `gitea` service to Nomad then it gets a `service: gitea` label in Prometheus. + +Additionally, I want to monitor Nomad itself and, well, it so happens that Nomad registers itself with Consul and so it gets collected as well. The only difference is that the `metrics_path` has to manually changed to `/v1/metrics`. + +## logs are, uh, not done + +Logging is incredibly import and it's on my TODO list to create a better log collection setup but for now I just use Nomad's log UI. + +![gitea_logs](/nomad_gitea_logs.png) + +## future todo + +Something that I'm excited to play with is using eBPF to construct fine-grained networking monitoring for the box to better monitor traffic flowing between tasks. There is a lot of [documentation](https://www.redhat.com/en/blog/monitoring-ebpf-based-metrics) for this - I just need to clear my calendar for it. + +Stay tuned for part 3 + +--- diff --git a/deploy/job.nomad b/deploy/job.nomad index e369560..a04cc06 100644 --- a/deploy/job.nomad +++ b/deploy/job.nomad @@ -4,8 +4,6 @@ job "blog" { group "blog" { network { - mode = "bridge" - port "http" { to = 80 } @@ -33,7 +31,7 @@ job "blog" { driver = "docker" config { - image = "git.howdoicomputer.lol/howdoicomputer/blog:5" + image = "git.howdoicomputer.lol/howdoicomputer/blog:7" ports = ["http"] } diff --git a/public/about/index.html b/public/about/index.html index 0f4b3cc..16deb4d 100644 --- a/public/about/index.html +++ b/public/about/index.html @@ -58,7 +58,7 @@
- +

howdoicomputer's blog

A dumping ground for ideas related to making, tomfoolery, and tomfoolery related to making

diff --git a/public/categories/index.html b/public/categories/index.html index eb3524a..771ad2e 100644 --- a/public/categories/index.html +++ b/public/categories/index.html @@ -52,7 +52,7 @@
- +

howdoicomputer's blog

A dumping ground for ideas related to making, tomfoolery, and tomfoolery related to making

diff --git a/public/favicon-16x16.png b/public/favicon-16x16.png new file mode 100644 index 0000000..7087bb7 Binary files /dev/null and b/public/favicon-16x16.png differ diff --git a/public/favicon-32x32.png b/public/favicon-32x32.png new file mode 100644 index 0000000..3f3b65d Binary files /dev/null and b/public/favicon-32x32.png differ diff --git a/public/favicon.ico b/public/favicon.ico new file mode 100644 index 0000000..bbbcdd9 Binary files /dev/null and b/public/favicon.ico differ diff --git a/public/index.html b/public/index.html index bce80ea..29d65d9 100644 --- a/public/index.html +++ b/public/index.html @@ -2,7 +2,7 @@ - howdoicomputer – howdoicomputer + howdoicomputer – howdoicomputer @@ -41,10 +41,15 @@
    - -
  • Posts +
  • Homelab 2: Monitoring Boogaloo
    - + How metrics and logs work for my homelab +
    +
    • + +
  • Homelab 1: Genesis +
    + First part of a multi-part series about my current homelab
    • @@ -55,7 +60,7 @@
    - +

    howdoicomputer's blog

    A dumping ground for ideas related to making, tomfoolery, and tomfoolery related to making

    diff --git a/public/index.xml b/public/index.xml index a5639ab..e1fe562 100644 --- a/public/index.xml +++ b/public/index.xml @@ -6,9 +6,18 @@ Recent content on howdoicomputer Hugo -- gohugo.io en-us - Sun, 27 Aug 2023 18:36:47 -0700 + Fri, 01 Sep 2023 21:15:40 -0700 - Homelab: Part 1 + Homelab 2: Monitoring Boogaloo + https://howdoicomputer.lol/posts/homelab-2/ + Fri, 01 Sep 2023 21:15:40 -0700 + + https://howdoicomputer.lol/posts/homelab-2/ + How metrics and logs work for my homelab + + + + Homelab 1: Genesis https://howdoicomputer.lol/posts/homelab-1/ Sun, 27 Aug 2023 18:36:47 -0700 diff --git a/public/node_exp_dashboard.png b/public/node_exp_dashboard.png new file mode 100644 index 0000000..1c95abe Binary files /dev/null and b/public/node_exp_dashboard.png differ diff --git a/public/nomad_gitea_logs.png b/public/nomad_gitea_logs.png new file mode 100644 index 0000000..557ab15 Binary files /dev/null and b/public/nomad_gitea_logs.png differ diff --git a/public/posts/homelab-1/index.html b/public/posts/homelab-1/index.html index fb1a81e..09cc709 100644 --- a/public/posts/homelab-1/index.html +++ b/public/posts/homelab-1/index.html @@ -1,7 +1,7 @@ - Homelab: Part 1 – howdoicomputer + Homelab 1: Genesis – howdoicomputer @@ -39,13 +39,12 @@
    -

    Homelab: Part 1

    +

    Homelab 1: Genesis

    -

    This blog. The one that you are currently reading right now. Is running on a metal box that is just barely large enough to fit the average house cat. It cost ~$800 USD to build and has been christened Megamind. The HTTP request that you initiated when you visited https://blog.howdoicomputer.lol has returned digital bits from hardware that is more than likely a couple feet from where I’m sitting. Welcome!

    +

    This blog. The one that you are currently reading right now. Is running on a metal box that is just barely large enough to fit the average house cat. It cost ~$800 USD to build and has been christened Megamind. The HTTP request that you initiated when you visited https://howdoicomputer.lol has returned digital bits from hardware that is more than likely a couple feet from where I’m sitting. Welcome!

    yeah, but why

    -

    I like solving problems and these were mine:

    • I have a plethora of software projects that I’ve written that need a home. Discord bots, daemons, websites, etc.
    • I needed a place to host a blog. Hi!
      • @@ -77,7 +76,7 @@
    • The operating system is NixOS and I use my laptop to do flake based deployments to the underlying OS

    A secret that I’ve withheld from you so far is that Megamind isn’t my first homelab. A previous homelab that I built was a k3s cluster running on top of Raspbery Pis. It was quaint but got decommissioned when I moved apartments. I never set it up again and it currently lies in pieces in a drawer somewhere.

    -

    So why Nomad? Mostly because, at the time, I was leading an infrastructure initiative at my day job to use Nomad to orchestrate applications that are deployed to heterogenous IoT devices spread across distinct geographic regions. I wanted to dog food my own proposal. Also, Nomad is a lot simpler to use than Kubernetes. Nomad starts as just an orchestrator but gives you the option to introduce complexity later on through Vault and Consul. Every concept is almost immediately grokable and it has just been a pleasure to work with.

    +

    So why Nomad? Mostly because, at the time, I was leading an infrastructure initiative at my day job to use Nomad to orchestrate applications that are deployed to heterogenous IoT devices spread across distinct geographic regions. Nomad was a better fit than k8s for that specific use case because it was lighter-weight, easier to use, had less overhead (which was important because embedded devices don’t have a lot of resources), and could run non-containerized applications that needed deep access to hardware. And so I wanted to dog food my own proposal. Also, Nomad is a lot simpler to use than Kubernetes. Nomad starts as just an orchestrator but gives you the option to introduce complexity later on through Vault and Consul. Every concept is almost immediately grokable and it has just been a pleasure to work with.

    what’s runnin

    Here is what is running so far:

      @@ -128,12 +127,12 @@ tags = [ "traefik.enable=true", "traefik.http.routers.blog-https.tls=true", - "traefik.http.routers.blog-https.rule=Host(`blog.howdoicomputer.lol`)", + "traefik.http.routers.blog-https.rule=Host(`howdoicomputer.lol`)", "traefik.http.routers.blog-https.tls.certresolver=resolver", - "traefik.http.routers.blog-https.tls.domains[0].main=blog.howdoicomputer.lol", + "traefik.http.routers.blog-https.tls.domains[0].main=howdoicomputer.lol", "traefik.http.routers.blog-https.entrypoints=websecure", "traefik.http.routers.blog-http.entrypoints=web", - "traefik.http.routers.blog-http.rule=Host(`blog.howdoicomputer.lol`)", + "traefik.http.routers.blog-http.rule=Host(`howdoicomputer.lol`)", "traefik.http.routers.blog-http.middlewares=redirecthttps", "traefik.http.middlewares.redirecthttps.redirectscheme.scheme=https" ] @@ -169,8 +168,8 @@

      the future

      The future is bright! It has been incredibly fluid to roll out new services to the server. There are two additional features that I want: a CI/CD solution and an auth model for friends who want to use services.

      For CI/CD, I’ll most likely go with Drone. Really, I just want a UI for Dagger pipelines. For an auth model, eh, I am not sure. LDAP? There is also the possiblity of using Oauth2 and whitelisting accounts.

      -

      I also want to do a thorough lockdown of Gitea so that I can expose my code to the world.

      -

      Till next time.

      +

      For now though, my homelab is now an integral part of my life. For consumption, I use it to store a digital library of books, music, and movies. For productivity, the homelab is a cornerstone of developing applications that I want to expose to the public internet. It’s… neat.

      +

      Part 2

    @@ -180,7 +179,7 @@
    - +

    howdoicomputer's blog

    A dumping ground for ideas related to making, tomfoolery, and tomfoolery related to making

    diff --git a/public/posts/homelab-2/index.html b/public/posts/homelab-2/index.html new file mode 100644 index 0000000..b5c8f23 --- /dev/null +++ b/public/posts/homelab-2/index.html @@ -0,0 +1,161 @@ + + + + Homelab 2: Monitoring Boogaloo – howdoicomputer + + + + + + + + + + + + + + + + + + + +
    + +
    + +
    + +
    +
    +

    Homelab 2: Monitoring Boogaloo

    +
    + +
    +

    Every good production environment needs robust monitoring in-place before it could be considered production and I feel like my homelab shouldn’t be an exception.

    +

    With that in mind, I created a monitoring stack that I’ve used professionally and I’m pretty happy with it.

    +

    dash +Grafana dashboard for host resource metrics

    +

    stackem'

    +

    The different components of the system are thus:

    +
      +
    • Prometheus
      • +
    • Grafana
      • +
    • Prometheus’s node_exporter
      • +
    • Consul
      • +
    +

    Here is what I want to collect metrics for, from most critical to least:

    +
      +
    1. The base host resources. This includes available memory, CPU, disk space, network traffic, etc. Also includes the ZFS pool.
      2. +
    2. Nomad itself.
      3. +
    3. The services that are orchestrated by Nomad.
      4. +
    +

    tldr prometheus

    +

    I won’t go too deeply into how Prometheus works; I wouldn’t be able to do better than the official documentation. That being said, it’s worthwhile to do a quick summary:

    +
      +
    • Applications are responsible for presenting their own metrics over HTTP via a dedicated endpoint
      • +
    • Prometheus operates on a pull based model. As in, Prometheus will reach out and HTTP GET that endpoint for each application in order to “scrape” those metrics and store them
      • +
    • Prometheus has a massive amount of support for different types of service discovery for automatically registering scrape targets
      • +
    • Prometheus is a time series database (TSDB) - it is built to store linear, time oriented data that has dimensionality expressed through key-value pairs
      • +
    +

    base system monitoring

    +

    For monitoring the base system, the Prometheus project supports the collection of metrics for Linux based hosts through their node_exporter project. It’s a Golang binary that exposes an absolute treasure trove of data… including ZFS stats!

    +

    The exporter project recommends running it outside of a container as it needs a deep access to the host system and the isolation of containers are counterintiutive to that level of access.

    +

    While I could run the node_exporter via systemd, I instead opted to use the raw_exec driver for Nomad. There is a Nix package for installing the exporter so the job definition just relies on executing the binary itself. Doing it this way means I get visibility into the exporter process and logs via the Nomad dashboard.

    +
    task "node-exporter" {
+  driver = "raw_exec"
+
+  config {
+    command = "/run/current-system/sw/bin/node_exporter"
+  }
+}
+

    There is some security implications for enabling the raw_exec driver for Nomad as it will run every process under the same user that the Nomad client is running as - which is usually root. However, it was the path of least resistance and it’s a TODO to later use the exec driver - which uses cgroups and chroot to isolate the spawned process. This, again, is counterintuitive to the node exporter being able to collect data and would require me to make sure that the task allocation has the correct access to the right resources in order to facilitate complete data collection.

    +

    service discovery brings all the metrics to the yard

    +

    Prometheus has support for a plethora of service discovery methods that can be used to help it locate endpoints for scraping. The integrations range from using simple files to integrating with service meshes. For my homelab, I opted to use the consul service discovery integration because, well, that’s what I’m running.

    +

    Whenever a service is deployed to Nomad, it will register an IP address and port to Consul. Prometheus will then talk to Consul to get that address to construct a metrics endpoint and then start scraping metrics an a specified interval.

    +

    The configuration for Prometheus looks like this:

    +
    scrape_configs:
+  - job_name: 'service_metrics'
+    consul_sd_configs:
+      - server: '{{ env "NOMAD_IP_prometheus" }}:8500'
+    relabel_configs:
+      - source_labels: [__meta_consul_service]
+        target_label: service
+

    Since everything is on the same box, there is no authentication method for Consul and so the only necessary config is the server endpoint. The $NOMAD_IP_prometheus is the IP address for the Prometheus task - which is also the IP address for the homelab server itself because I make liberal use of the bridge networking type. As a quick note, bridge here means that all tasks share a networking namespace but ingress happens over a port bound to localhost for the Nomad client.

    +

    The relabel config exists to take the Consul service source label - in this case, the Consul service label is the name of the service that is running on Nomad - and create a target label of service so that every metric is properly tagged. This means that if I deploy a gitea service to Nomad then it gets a service: gitea label in Prometheus.

    +

    Additionally, I want to monitor Nomad itself and, well, it so happens that Nomad registers itself with Consul and so it gets collected as well. The only difference is that the metrics_path has to manually changed to /v1/metrics.

    +

    logs are, uh, not done

    +

    Logging is incredibly import and it’s on my TODO list to create a better log collection setup but for now I just use Nomad’s log UI.

    +

    gitea_logs

    +

    future todo

    +

    Something that I’m excited to play with is using eBPF to construct fine-grained networking monitoring for the box to better monitor traffic flowing between tasks. There is a lot of documentation for this - I just need to clear my calendar for it.

    +

    Stay tuned for part 3

    +
    + +
    +
    + +
    + +
    +
    +
    + +

    howdoicomputer's blog

    +

    A dumping ground for ideas related to making, tomfoolery, and tomfoolery related to making

    +
    + + + +
    +
    +
    + + +

    + + 2023-09-01 +

    + + + + +
    +
    + +

    + + + + + + + + + + + + +

    +

    +

    + +
    + +
    + + + diff --git a/public/posts/index.html b/public/posts/index.html index a1c9c42..ccbe00f 100644 --- a/public/posts/index.html +++ b/public/posts/index.html @@ -44,7 +44,10 @@ @@ -55,7 +58,7 @@
    - +

    howdoicomputer's blog

    A dumping ground for ideas related to making, tomfoolery, and tomfoolery related to making

    diff --git a/public/posts/index.xml b/public/posts/index.xml index 5be99a2..3b74fc2 100644 --- a/public/posts/index.xml +++ b/public/posts/index.xml @@ -6,9 +6,18 @@ Recent content in Posts on howdoicomputer Hugo -- gohugo.io en-us - Sun, 27 Aug 2023 18:36:47 -0700 + Fri, 01 Sep 2023 21:15:40 -0700 - Homelab: Part 1 + Homelab 2: Monitoring Boogaloo + https://howdoicomputer.lol/posts/homelab-2/ + Fri, 01 Sep 2023 21:15:40 -0700 + + https://howdoicomputer.lol/posts/homelab-2/ + How metrics and logs work for my homelab + + + + Homelab 1: Genesis https://howdoicomputer.lol/posts/homelab-1/ Sun, 27 Aug 2023 18:36:47 -0700 diff --git a/public/resume.pdf b/public/resume.pdf new file mode 100644 index 0000000..aa0fde9 Binary files /dev/null and b/public/resume.pdf differ diff --git a/public/sitemap.xml b/public/sitemap.xml index 95c2f9a..d422d98 100644 --- a/public/sitemap.xml +++ b/public/sitemap.xml @@ -3,28 +3,40 @@ xmlns:xhtml="http://www.w3.org/1999/xhtml"> https://howdoicomputer.lol/tags/consul/ - 2023-08-27T18:36:47-07:00 + 2023-09-01T21:15:40-07:00 + + https://howdoicomputer.lol/tags/grafana/ + 2023-09-01T21:15:40-07:00 + + https://howdoicomputer.lol/posts/homelab-2/ + 2023-09-01T21:15:40-07:00 + + https://howdoicomputer.lol/ + 2023-09-01T21:15:40-07:00 + + https://howdoicomputer.lol/tags/monitoring/ + 2023-09-01T21:15:40-07:00 + + https://howdoicomputer.lol/tags/nomad/ + 2023-09-01T21:15:40-07:00 + + https://howdoicomputer.lol/posts/ + 2023-09-01T21:15:40-07:00 + + https://howdoicomputer.lol/tags/prometheus/ + 2023-09-01T21:15:40-07:00 + + https://howdoicomputer.lol/tags/ + 2023-09-01T21:15:40-07:00 https://howdoicomputer.lol/tags/homelab/ 2023-08-27T18:36:47-07:00 https://howdoicomputer.lol/posts/homelab-1/ 2023-08-27T18:36:47-07:00 - - https://howdoicomputer.lol/ - 2023-08-27T18:36:47-07:00 https://howdoicomputer.lol/tags/nix/ 2023-08-27T18:36:47-07:00 - - https://howdoicomputer.lol/tags/nomad/ - 2023-08-27T18:36:47-07:00 - - https://howdoicomputer.lol/posts/ - 2023-08-27T18:36:47-07:00 - - https://howdoicomputer.lol/tags/ - 2023-08-27T18:36:47-07:00 https://howdoicomputer.lol/about/ diff --git a/public/tags/consul/index.html b/public/tags/consul/index.html index 3abc2c8..92edbf2 100644 --- a/public/tags/consul/index.html +++ b/public/tags/consul/index.html @@ -44,7 +44,10 @@ @@ -55,7 +58,7 @@
    - +

    howdoicomputer's blog

    A dumping ground for ideas related to making, tomfoolery, and tomfoolery related to making

    diff --git a/public/tags/consul/index.xml b/public/tags/consul/index.xml index 3496e8a..d5a1753 100644 --- a/public/tags/consul/index.xml +++ b/public/tags/consul/index.xml @@ -6,9 +6,18 @@ Recent content in Consul on howdoicomputer Hugo -- gohugo.io en-us - Sun, 27 Aug 2023 18:36:47 -0700 + Fri, 01 Sep 2023 21:15:40 -0700 - Homelab: Part 1 + Homelab 2: Monitoring Boogaloo + https://howdoicomputer.lol/posts/homelab-2/ + Fri, 01 Sep 2023 21:15:40 -0700 + + https://howdoicomputer.lol/posts/homelab-2/ + How metrics and logs work for my homelab + + + + Homelab 1: Genesis https://howdoicomputer.lol/posts/homelab-1/ Sun, 27 Aug 2023 18:36:47 -0700 diff --git a/public/tags/grafana/index.html b/public/tags/grafana/index.html new file mode 100644 index 0000000..358e75c --- /dev/null +++ b/public/tags/grafana/index.html @@ -0,0 +1,95 @@ + + + + Grafana – howdoicomputer + + + + + + + + + + + + + + + + + + + +
    + +
    + +
    + +
    +

    Grafana

    + + + + + + +
    + +
    +
    +
    + +

    howdoicomputer's blog

    +

    A dumping ground for ideas related to making, tomfoolery, and tomfoolery related to making

    +
    + + + +
    +
    +
    +
    +
    + +

    + + + + + + + + + + + + +

    +

    +

    + +
    + +
    + + + diff --git a/public/tags/grafana/index.xml b/public/tags/grafana/index.xml new file mode 100644 index 0000000..4e72a48 --- /dev/null +++ b/public/tags/grafana/index.xml @@ -0,0 +1,20 @@ + + + + Grafana on howdoicomputer + https://howdoicomputer.lol/tags/grafana/ + Recent content in Grafana on howdoicomputer + Hugo -- gohugo.io + en-us + Fri, 01 Sep 2023 21:15:40 -0700 + + Homelab 2: Monitoring Boogaloo + https://howdoicomputer.lol/posts/homelab-2/ + Fri, 01 Sep 2023 21:15:40 -0700 + + https://howdoicomputer.lol/posts/homelab-2/ + How metrics and logs work for my homelab + + + + diff --git a/public/tags/homelab/index.html b/public/tags/homelab/index.html index d383ee9..876c596 100644 --- a/public/tags/homelab/index.html +++ b/public/tags/homelab/index.html @@ -44,7 +44,7 @@ @@ -55,7 +55,7 @@
    - +

    howdoicomputer's blog

    A dumping ground for ideas related to making, tomfoolery, and tomfoolery related to making

    diff --git a/public/tags/homelab/index.xml b/public/tags/homelab/index.xml index e561df3..3df28cc 100644 --- a/public/tags/homelab/index.xml +++ b/public/tags/homelab/index.xml @@ -8,7 +8,7 @@ en-us Sun, 27 Aug 2023 18:36:47 -0700 - Homelab: Part 1 + Homelab 1: Genesis https://howdoicomputer.lol/posts/homelab-1/ Sun, 27 Aug 2023 18:36:47 -0700 diff --git a/public/tags/index.html b/public/tags/index.html index 6341fd1..ea1792d 100644 --- a/public/tags/index.html +++ b/public/tags/index.html @@ -47,15 +47,24 @@
  • Consul
    • +
  • Grafana
    • + + +
  • Monitoring
    • + + +
  • Nomad
    • + + +
  • Prometheus
    • + +
  • Homelab
  • Nix
    • -
  • Nomad
    • - -
@@ -64,7 +73,7 @@
- +

howdoicomputer's blog

A dumping ground for ideas related to making, tomfoolery, and tomfoolery related to making

diff --git a/public/tags/index.xml b/public/tags/index.xml index fbca43c..babd4ff 100644 --- a/public/tags/index.xml +++ b/public/tags/index.xml @@ -6,16 +6,52 @@ Recent content in Tags on howdoicomputer Hugo -- gohugo.io en-us - Sun, 27 Aug 2023 18:36:47 -0700 + Fri, 01 Sep 2023 21:15:40 -0700 Consul https://howdoicomputer.lol/tags/consul/ - Sun, 27 Aug 2023 18:36:47 -0700 + Fri, 01 Sep 2023 21:15:40 -0700 https://howdoicomputer.lol/tags/consul/ + + Grafana + https://howdoicomputer.lol/tags/grafana/ + Fri, 01 Sep 2023 21:15:40 -0700 + + https://howdoicomputer.lol/tags/grafana/ + + + + + Monitoring + https://howdoicomputer.lol/tags/monitoring/ + Fri, 01 Sep 2023 21:15:40 -0700 + + https://howdoicomputer.lol/tags/monitoring/ + + + + + Nomad + https://howdoicomputer.lol/tags/nomad/ + Fri, 01 Sep 2023 21:15:40 -0700 + + https://howdoicomputer.lol/tags/nomad/ + + + + + Prometheus + https://howdoicomputer.lol/tags/prometheus/ + Fri, 01 Sep 2023 21:15:40 -0700 + + https://howdoicomputer.lol/tags/prometheus/ + + + Homelab https://howdoicomputer.lol/tags/homelab/ @@ -34,14 +70,5 @@ - - Nomad - https://howdoicomputer.lol/tags/nomad/ - Sun, 27 Aug 2023 18:36:47 -0700 - - https://howdoicomputer.lol/tags/nomad/ - - - diff --git a/public/tags/monitoring/index.html b/public/tags/monitoring/index.html new file mode 100644 index 0000000..50c3035 --- /dev/null +++ b/public/tags/monitoring/index.html @@ -0,0 +1,95 @@ + + + + Monitoring – howdoicomputer + + + + + + + + + + + + + + + + + + + +
+ +
+ +
+ +
+

Monitoring

+ + + + + + +
+ +
+
+
+ +

howdoicomputer's blog

+

A dumping ground for ideas related to making, tomfoolery, and tomfoolery related to making

+
+ + + +
+
+
+
+
+ +

+ + + + + + + + + + + + +

+

+

+ +
+ +
+ + + diff --git a/public/tags/monitoring/index.xml b/public/tags/monitoring/index.xml new file mode 100644 index 0000000..7c750b7 --- /dev/null +++ b/public/tags/monitoring/index.xml @@ -0,0 +1,20 @@ + + + + Monitoring on howdoicomputer + https://howdoicomputer.lol/tags/monitoring/ + Recent content in Monitoring on howdoicomputer + Hugo -- gohugo.io + en-us + Fri, 01 Sep 2023 21:15:40 -0700 + + Homelab 2: Monitoring Boogaloo + https://howdoicomputer.lol/posts/homelab-2/ + Fri, 01 Sep 2023 21:15:40 -0700 + + https://howdoicomputer.lol/posts/homelab-2/ + How metrics and logs work for my homelab + + + + diff --git a/public/tags/nix/index.html b/public/tags/nix/index.html index 3e1e639..f8d7f33 100644 --- a/public/tags/nix/index.html +++ b/public/tags/nix/index.html @@ -44,7 +44,7 @@ @@ -55,7 +55,7 @@
- +

howdoicomputer's blog

A dumping ground for ideas related to making, tomfoolery, and tomfoolery related to making

diff --git a/public/tags/nix/index.xml b/public/tags/nix/index.xml index 978f41a..b038810 100644 --- a/public/tags/nix/index.xml +++ b/public/tags/nix/index.xml @@ -8,7 +8,7 @@ en-us Sun, 27 Aug 2023 18:36:47 -0700 - Homelab: Part 1 + Homelab 1: Genesis https://howdoicomputer.lol/posts/homelab-1/ Sun, 27 Aug 2023 18:36:47 -0700 diff --git a/public/tags/nomad/index.html b/public/tags/nomad/index.html index 3990ba1..1e258cb 100644 --- a/public/tags/nomad/index.html +++ b/public/tags/nomad/index.html @@ -44,7 +44,10 @@ @@ -55,7 +58,7 @@
- +

howdoicomputer's blog

A dumping ground for ideas related to making, tomfoolery, and tomfoolery related to making

diff --git a/public/tags/nomad/index.xml b/public/tags/nomad/index.xml index 5236f11..f8e04da 100644 --- a/public/tags/nomad/index.xml +++ b/public/tags/nomad/index.xml @@ -6,9 +6,18 @@ Recent content in Nomad on howdoicomputer Hugo -- gohugo.io en-us - Sun, 27 Aug 2023 18:36:47 -0700 + Fri, 01 Sep 2023 21:15:40 -0700 - Homelab: Part 1 + Homelab 2: Monitoring Boogaloo + https://howdoicomputer.lol/posts/homelab-2/ + Fri, 01 Sep 2023 21:15:40 -0700 + + https://howdoicomputer.lol/posts/homelab-2/ + How metrics and logs work for my homelab + + + + Homelab 1: Genesis https://howdoicomputer.lol/posts/homelab-1/ Sun, 27 Aug 2023 18:36:47 -0700 diff --git a/public/tags/prometheus/index.html b/public/tags/prometheus/index.html new file mode 100644 index 0000000..b7db2d9 --- /dev/null +++ b/public/tags/prometheus/index.html @@ -0,0 +1,95 @@ + + + + Prometheus – howdoicomputer + + + + + + + + + + + + + + + + + + + +
+ +
+ +
+ +
+

Prometheus

+ + + + + + +
+ +
+
+
+ +

howdoicomputer's blog

+

A dumping ground for ideas related to making, tomfoolery, and tomfoolery related to making

+
+ + + +
+
+
+
+
+ +

+ + + + + + + + + + + + +

+

+

+ +
+ +
+ + + diff --git a/public/tags/prometheus/index.xml b/public/tags/prometheus/index.xml new file mode 100644 index 0000000..d541420 --- /dev/null +++ b/public/tags/prometheus/index.xml @@ -0,0 +1,20 @@ + + + + Prometheus on howdoicomputer + https://howdoicomputer.lol/tags/prometheus/ + Recent content in Prometheus on howdoicomputer + Hugo -- gohugo.io + en-us + Fri, 01 Sep 2023 21:15:40 -0700 + + Homelab 2: Monitoring Boogaloo + https://howdoicomputer.lol/posts/homelab-2/ + Fri, 01 Sep 2023 21:15:40 -0700 + + https://howdoicomputer.lol/posts/homelab-2/ + How metrics and logs work for my homelab + + + + diff --git a/static/node_exp_dashboard.png b/static/node_exp_dashboard.png new file mode 100644 index 0000000..1c95abe Binary files /dev/null and b/static/node_exp_dashboard.png differ diff --git a/static/nomad_gitea_logs.png b/static/nomad_gitea_logs.png new file mode 100644 index 0000000..557ab15 Binary files /dev/null and b/static/nomad_gitea_logs.png differ diff --git a/static/resume.pdf b/static/resume.pdf new file mode 100644 index 0000000..aa0fde9 Binary files /dev/null and b/static/resume.pdf differ