Create new service with MemoryHigh and MemoryMax directives.

$ systemctl edit --force --full memory.service
[Unit]
Description=Simple service to test memory limit.

[Service]
ExecStart=/root/memory.sh
MemoryHigh=1M
MemoryMax=2M

[Install]
WantedBy=multi-user.target

The content of /root/memory.sh:

#!/bin/bash

echo $(date) > /tmp/test.log
a=()

for (( a=1; a<=10; a++ ))
do
    echo Loop $a >> /tmp/test.log
    for (( c=1; c<=600000; c++ ))
    do
        a+=( "abcdefghijklmnopqrstquvxyabcdefghijklmnopqrstquvxyzabcdefghijklmnopqrstquvxyzabcdefghijklmnopqrstquvxyzabcdefghijklmnopqrstquvxyzabcdefghijklmnopqrstquvxyzabcdefghijklmnopqrstquvxyzabcdefghijklmnopqrstquvxyzabcdefghijklmnopqrstquvxyzabcdefghijklmnopqrstquvxyzabcdefghijklmnopqrstquvxyzzabcdefghijklmnopqrstquvxyabcdefghijklmnopqrstquvxyzabcdefghijklmnopqrstquvxyzabcdefghijklmnopqrstquvxyzabcdefghijklmnopqrstquvxyzabcdefghijklmnopqrstquvxyzabcdefghijklmnopqrstquvxyzabcdefghijklmnopqrstquvxyzabcdefghijklmnopqrstquvxyzabcdefghijklmnopqrstquvxyzabcdefghijklmnopqrstquvxyzzabcdefghijklmnopqrstquvxyabcdefghijklmnopqrstquvxyzabcdefghijklmnopqrstquvxyzabcdefghijklmnopqrstquvxyzabcdefghijklmnopqrstquvxyzabcdefghijklmnopqrstquvxyzabcdefghijklmnopqrstquvxyzabcdefghijklmnopqrstquvxyzabcdefghijklmnopqrstquvxyzabcdefghijklmnopqrstquvxyzabcdefghijklmnopqrstquvxyzz" )
    done
done

sleep 10

Start the service:

root@tuxedo:/etc/systemd/system# systemctl daemon-reload
root@tuxedo:/etc/systemd/system# systemctl enable --now memory
root@tuxedo:/etc/systemd/system# systemctl status memory
● memory.service - Simple service to test memory limit.
Loaded: loaded (/etc/systemd/system/memory.service; enabled; vendor preset: enabled)
Active: active (running) since Thu 2022-09-01 22:26:27 CEST; 9s ago
Main PID: 14675 (memory.sh)
Tasks: 1 (limit: 76224)
Memory: 1.9M (high: 1.0M max: 2.0M)
CGroup: /system.slice/_memory.service
└─14675 /bin/bash /root/memory.sh

wrz 01 22:26:27 tuxedo systemd[1]: Started Simple service to test memory limit..

After a while

root@tuxedo:/etc/systemd/system# systemctl status memory
● memory.service - Simple service to test memory limit.
Loaded: loaded (/etc/systemd/system/memory.service; enabled; vendor preset: enabled)
Active: failed (Result: signal) since Thu 2022-09-01 22:27:31 CEST; 8s ago
Process: 14675 ExecStart=/root/memory.sh (code=killed, signal=KILL)
Main PID: 14675 (code=killed, signal=KILL)

wrz 01 22:26:27 tuxedo systemd[1]: Started Simple service to test memory limit..
wrz 01 22:27:31 tuxedo systemd[1]: memory.service: Main process exited, code=killed, status=9/KILL
wrz 01 22:27:31 tuxedo systemd[1]: memory.service: Failed with result 'signal'.

And in the dmesg:

$ dmesg
[ 5679.682307] Tasks state (memory values in pages):
[ 5679.682308] [  pid  ]   uid  tgid total_vm      rss pgtables_bytes swapents oom_score_adj name
[ 5679.682310] [  14675]     0 14675   202158      862  1646592   199134             0 memory.sh
[ 5679.682316] oom-kill:constraint=CONSTRAINT_MEMCG,nodemask=(null),cpuset=/,mems_allowed=0,oom_memcg=/system.slice/_memory.service,task_memcg=/system.slice/_memory.service,task=memory.sh,pid=14675,uid=0
[ 5679.682330] Memory cgroup out of memory: Killed process 14675 (memory.sh) total-vm:808632kB, anon-rss:0kB, file-rss:3448kB, shmem-rss:0kB, UID:0 pgtables:1608kB oom_score_adj:0