Simplest possible service that runs /root/process.sh script:

root@tuxedo:/etc/systemd/system# cat process.service

[Unit]
Description=Simple service to test process limit.

[Service]
ExecStart=/root/process.sh

[Install]
WantedBy=multi-user.target

The content of /root/process.sh:

#!/bin/bash

echo $(date) >> /tmp/test.log
for (( c=1; c<=10; c++ ))
do
sleep 10 &
done

sleep 20

Start the service:

root@tuxedo:/etc/systemd/system# systemctl daemon-reload
root@tuxedo:/etc/systemd/system# systemctl enable --now process
root@tuxedo:/etc/systemd/system# systemctl status process
● process.service - Simple service to test process limit.
Loaded: loaded (/etc/systemd/system/process.service; enabled; vendor preset: enabled)
Active: active (running) since Tue 2022-08-30 22:59:03 CEST; 2s ago
Main PID: 18596 (process.sh)
Tasks: 12 (limit: 76224)
Memory: 2.3M
CGroup: /system.slice/process.service
├─18596 /bin/bash /root/process.sh
├─18598 sleep 10
├─18599 sleep 10
├─18600 sleep 10
├─18601 sleep 10
├─18602 sleep 10
├─18603 sleep 10
├─18604 sleep 10
├─18605 sleep 10
├─18606 sleep 10
├─18607 sleep 10
└─18608 sleep 20

sie 30 22:59:03 tuxedo systemd[1]: Started Simple service to test process limit..

Now add TasksMax=5 into [Service] section:

[Unit]
Description=Simple service to test process limit.

[Service]
ExecStart=/root/process.sh
TasksMax=5

[Install]
WantedBy=multi-user.target

and restart the service:

root@tuxedo:/etc/systemd/system# systemctl daemon-reload
root@tuxedo:/etc/systemd/system# systemctl restart process
root@tuxedo:/etc/systemd/system# systemctl status process
● process.service - Simple service to test process limit.
Loaded: loaded (/etc/systemd/system/process.service; enabled; vendor preset: enabled)
Active: active (running) since Tue 2022-08-30 23:04:20 CEST; 9s ago
Main PID: 18961 (process.sh)
Tasks: 5 (limit: 5)
Memory: 1.0M
CGroup: /system.slice/process.service
├─18961 /bin/bash /root/process.sh
├─18963 sleep 10
├─18964 sleep 10
├─18965 sleep 10
└─18966 sleep 10

sie 30 23:04:20 tuxedo systemd[1]: Started Simple service to test process limit..
sie 30 23:04:20 tuxedo process.sh[18961]: /root/process.sh: fork: retry: Resource temporarily unavailab>
sie 30 23:04:21 tuxedo process.sh[18961]: /root/process.sh: fork: retry: Resource temporarily unavailab>
sie 30 23:04:23 tuxedo process.sh[18961]: /root/process.sh: fork: retry: Resource temporarily unavailab>
sie 30 23:04:27 tuxedo process.sh[18961]: /root/process.sh: fork: retry: Resource temporarily unavailab>
sie 30 23:04:30 tuxedo process.sh[18961]: /root/process.sh: fork: Interrupted system call
sie 30 23:04:30 tuxedo systemd[1]: process.service: Main process exited, code=exited, status=254/n/a
sie 30 23:04:30 tuxedo systemd[1]: process.service: Failed with result 'exit-code'.